From a77f34b7ba55556de8cf24ef19b02b48bcfc153c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 11:07:47 +0100 Subject: [PATCH 01/10] Add a processing step to retrieve page counts --- .../docspell/extract/pdfbox/PdfMetaData.scala | 8 +- .../extract/pdfbox/PdfboxExtract.scala | 38 +++++---- .../joex/process/AttachmentPageCount.scala | 83 +++++++++++++++++++ .../docspell/joex/process/ProcessItem.scala | 1 + .../db/migration/h2/V1.11.0__pdf_pages.sql | 2 + .../migration/mariadb/V1.11.0__pdf_pages.sql | 2 + .../postgresql/V1.11.0__pdf_pages.sql | 2 + .../store/records/RAttachmentMeta.scala | 16 +++- 8 files changed, 128 insertions(+), 24 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala create mode 100644 modules/store/src/main/resources/db/migration/h2/V1.11.0__pdf_pages.sql create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.11.0__pdf_pages.sql create mode 100644 modules/store/src/main/resources/db/migration/postgresql/V1.11.0__pdf_pages.sql diff --git a/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfMetaData.scala b/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfMetaData.scala index 4663d1c8..eb450ae9 100644 --- a/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfMetaData.scala +++ b/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfMetaData.scala @@ -8,7 +8,8 @@ final case class PdfMetaData( subject: Option[String], keywords: Option[String], creator: Option[String], - creationDate: Option[Timestamp] + creationDate: Option[Timestamp], + pageCount: Int ) { def isEmpty: Boolean = @@ -17,7 +18,8 @@ final case class PdfMetaData( subject.isEmpty && keywords.isEmpty && creator.isEmpty && - creationDate.isEmpty + creationDate.isEmpty && + pageCount <= 0 def nonEmpty: Boolean = !isEmpty @@ -36,5 +38,5 @@ final case class PdfMetaData( } object PdfMetaData { - val empty = PdfMetaData(None, None, None, None, None, None) + val empty = PdfMetaData(None, None, None, None, None, None, 0) } diff --git a/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfboxExtract.scala b/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfboxExtract.scala index def9c8ee..d3267503 100644 --- a/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfboxExtract.scala +++ b/modules/extract/src/main/scala/docspell/extract/pdfbox/PdfboxExtract.scala @@ -20,21 +20,23 @@ object PdfboxExtract { def getTextAndMetaData[F[_]: Sync]( data: Stream[F, Byte] ): F[Either[Throwable, (Text, Option[PdfMetaData])]] = - data.compile - .to(Array) - .map(bytes => - Using(PDDocument.load(bytes)) { doc => - for { - txt <- readText(doc) - md <- readMetaData(doc) - } yield (txt, Some(md).filter(_.nonEmpty)) - }.toEither.flatten - ) + PdfLoader + .withDocumentStream(data) { doc => + (for { + txt <- readText(doc) + md <- readMetaData(doc) + } yield (txt, Some(md).filter(_.nonEmpty))).pure[F] + } + .attempt + .map(_.flatten) def getText[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, Text]] = - data.compile - .to(Array) - .map(bytes => Using(PDDocument.load(bytes))(readText).toEither.flatten) + PdfLoader + .withDocumentStream(data) { doc => + readText(doc).pure[F] + } + .attempt + .map(_.flatten) def getText(is: InputStream): Either[Throwable, Text] = Using(PDDocument.load(is))(readText).toEither.flatten @@ -51,9 +53,10 @@ object PdfboxExtract { }.toEither def getMetaData[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, PdfMetaData]] = - data.compile - .to(Array) - .map(bytes => Using(PDDocument.load(bytes))(readMetaData).toEither.flatten) + PdfLoader + .withDocumentStream(data)(doc => readMetaData(doc).pure[F]) + .attempt + .map(_.flatten) def getMetaData(is: InputStream): Either[Throwable, PdfMetaData] = Using(PDDocument.load(is))(readMetaData).toEither.flatten @@ -73,7 +76,8 @@ object PdfboxExtract { mkValue(info.getSubject), mkValue(info.getKeywords), mkValue(info.getCreator), - Option(info.getCreationDate).map(c => Timestamp(c.toInstant)) + Option(info.getCreationDate).map(c => Timestamp(c.toInstant)), + doc.getNumberOfPages() ) }.toEither } diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala new file mode 100644 index 00000000..c1dbe7e4 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala @@ -0,0 +1,83 @@ +package docspell.joex.process + +import cats.Functor +import cats.data.OptionT +import cats.effect._ +import cats.implicits._ +import fs2.Stream + +import docspell.common._ +import docspell.extract.pdfbox.PdfMetaData +import docspell.extract.pdfbox.PdfboxExtract +import docspell.joex.scheduler._ +import docspell.store.records.RAttachment +import docspell.store.records._ +import docspell.store.syntax.MimeTypes._ + +import bitpeace.{Mimetype, RangeDef} + +/** Goes through all attachments that must be already converted into a + * pdf. If it is a pdf, the number of pages are retrieved and stored + * in the attachment metadata. + */ +object AttachmentPageCount { + + def apply[F[_]: Sync: ContextShift]()( + item: ItemData + ): Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + for { + _ <- ctx.logger.info( + s"Retrieving page count for ${item.attachments.size} files…" + ) + _ <- item.attachments + .traverse(createPageCount(ctx)) + .attempt + .flatMap { + case Right(_) => ().pure[F] + case Left(ex) => + ctx.logger.error(ex)( + s"Retrieving page counts failed, continuing without it." + ) + } + } yield item + } + + def createPageCount[F[_]: Sync]( + ctx: Context[F, _] + )(ra: RAttachment): F[Option[PdfMetaData]] = + findMime[F](ctx)(ra).flatMap { + case MimeType.PdfMatch(_) => + PdfboxExtract.getMetaData(loadFile(ctx)(ra)).flatMap { + case Right(md) => + updatePageCount(ctx, md, ra).map(_.some) + case Left(ex) => + ctx.logger.warn(s"Error obtaining pages count: ${ex.getMessage}") *> + (None: Option[PdfMetaData]).pure[F] + } + + case _ => + (None: Option[PdfMetaData]).pure[F] + } + + private def updatePageCount[F[_]: Sync]( + ctx: Context[F, _], + md: PdfMetaData, + ra: RAttachment + ): F[PdfMetaData] = + ctx.store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some)) *> md + .pure[F] + + def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = + OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) + .map(_.mimetype) + .getOrElse(Mimetype.`application/octet-stream`) + .map(_.toLocal) + + def loadFile[F[_]](ctx: Context[F, _])(ra: RAttachment): Stream[F, Byte] = + ctx.store.bitpeace + .get(ra.fileId.id) + .unNoneTerminate + .through(ctx.store.bitpeace.fetchData2(RangeDef.all)) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 8caf25fb..56f3cd33 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -55,6 +55,7 @@ object ProcessItem { .flatMap(Task.setProgress(progress._1)) .flatMap(TextExtraction(cfg.extraction, fts)) .flatMap(AttachmentPreview(cfg.convert, cfg.extraction.preview)) + .flatMap(AttachmentPageCount()) .flatMap(Task.setProgress(progress._2)) .flatMap(analysisOnly[F](cfg, analyser, regexNer)) .flatMap(Task.setProgress(progress._3)) diff --git a/modules/store/src/main/resources/db/migration/h2/V1.11.0__pdf_pages.sql b/modules/store/src/main/resources/db/migration/h2/V1.11.0__pdf_pages.sql new file mode 100644 index 00000000..ca347ea6 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.11.0__pdf_pages.sql @@ -0,0 +1,2 @@ +ALTER TABLE "attachmentmeta" +ADD COLUMN "page_count" smallint; diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.11.0__pdf_pages.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.11.0__pdf_pages.sql new file mode 100644 index 00000000..fd580127 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.11.0__pdf_pages.sql @@ -0,0 +1,2 @@ +ALTER TABLE `attachmentmeta` +ADD COLUMN (`page_count` SMALLINT); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.11.0__pdf_pages.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.11.0__pdf_pages.sql new file mode 100644 index 00000000..ca347ea6 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.11.0__pdf_pages.sql @@ -0,0 +1,2 @@ +ALTER TABLE "attachmentmeta" +ADD COLUMN "page_count" smallint; diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index d1cb79ea..833bfeca 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -13,17 +13,21 @@ case class RAttachmentMeta( id: Ident, //same as RAttachment.id content: Option[String], nerlabels: List[NerLabel], - proposals: MetaProposalList + proposals: MetaProposalList, + pages: Option[Int] ) { def setContentIfEmpty(txt: Option[String]): RAttachmentMeta = if (content.forall(_.trim.isEmpty)) copy(content = txt) else this + + def withPageCount(count: Option[Int]): RAttachmentMeta = + copy(pages = count) } object RAttachmentMeta { def empty(attachId: Ident) = - RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty) + RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty, None) val table = fr"attachmentmeta" @@ -32,7 +36,8 @@ object RAttachmentMeta { val content = Column("content") val nerlabels = Column("nerlabels") val proposals = Column("itemproposals") - val all = List(id, content, nerlabels, proposals) + val pages = Column("page_count") + val all = List(id, content, nerlabels, proposals, pages) } import Columns._ @@ -40,7 +45,7 @@ object RAttachmentMeta { insertRow( table, all, - fr"${v.id},${v.content},${v.nerlabels},${v.proposals}" + fr"${v.id},${v.content},${v.nerlabels},${v.proposals},${v.pages}" ).update.run def exists(attachId: Ident): ConnectionIO[Boolean] = @@ -84,6 +89,9 @@ object RAttachmentMeta { ) ).update.run + def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] = + updateRow(table, id.is(mid), pages.setTo(pageCount)).update.run + def delete(attachId: Ident): ConnectionIO[Int] = deleteFrom(table, id.is(attachId)).update.run } From 8c08bf233d487835cab232e1053b2c3e441ae24a Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 14:24:28 +0100 Subject: [PATCH 02/10] Amend search results with attachment info This uses again another query per item to retrieve some information about each attachment already in the search results. --- .../src/main/resources/docspell-openapi.yml | 38 +++++++++++++++---- .../restserver/conv/Conversions.scala | 8 +++- .../scala/docspell/store/queries/QItem.scala | 35 ++++++++++++++++- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 3ef90ff3..fbba6e89 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1291,10 +1291,11 @@ paths: summary: Search for items. description: | Search for items given a search form. The results are grouped - by month and are sorted by item date (newest first). Tags are - *not* resolved. The results will always contain an empty list - for item tags. Use `/searchWithTags` to also retrieve all tags - of an item. + by month and are sorted by item date (newest first). Tags and + attachments are *not* resolved. The results will always + contain an empty list for item tags and attachments. Use + `/searchWithTags` to also retrieve all tags and a list of + attachments of an item. The `fulltext` field can be used to restrict the results by using full-text search in the documents contents. @@ -1318,9 +1319,10 @@ paths: summary: Search for items. description: | Search for items given a search form. The results are grouped - by month by default. For each item, its tags are also - returned. This uses more queries and is therefore slower, but - returns all tags to an item. + by month by default. For each item, its tags and attachments + are also returned. This uses more queries and is therefore + slower, but returns all tags to an item as well as their + attachments with some minor details. The `fulltext` field can be used to restrict the results by using full-text search in the documents contents. @@ -4703,6 +4705,10 @@ components: fileCount: type: integer format: int32 + attachments: + type: array + items: + $ref: "#/components/schemas/AttachmentLight" tags: type: array items: @@ -4720,6 +4726,24 @@ components: type: array items: $ref: "#/components/schemas/HighlightEntry" + AttachmentLight: + description: | + Some little details about an attachment. + required: + - id + - position + properties: + id: + type: string + format: ident + position: + type: integer + format: int32 + name: + type: string + pageCount: + type: integer + format: int32 HighlightEntry: description: | Highlighting information for a single field (maybe attachment diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 3b40c1d8..fc0ccb75 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -22,6 +22,7 @@ import bitpeace.FileMeta import org.http4s.headers.`Content-Type` import org.http4s.multipart.Multipart import org.log4s.Logger +import docspell.store.queries.QItem trait Conversions { @@ -204,6 +205,7 @@ trait Conversions { i.folder.map(mkIdName), i.fileCount, Nil, + Nil, i.notes, Nil ) @@ -215,7 +217,11 @@ trait Conversions { } def mkItemLightWithTags(i: OItemSearch.ListItemWithTags): ItemLight = - mkItemLight(i.item).copy(tags = i.tags.map(mkTag)) + mkItemLight(i.item) + .copy(tags = i.tags.map(mkTag), attachments = i.attachments.map(mkAttachmentLight)) + + private def mkAttachmentLight(qa: QItem.AttachmentLight): AttachmentLight = + AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount) def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = { val il = mkItemLightWithTags(i.item) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index bce5f836..7f768f93 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -443,7 +443,17 @@ object QItem { from.query[ListItem].stream } - case class ListItemWithTags(item: ListItem, tags: List[RTag]) + case class AttachmentLight( + id: Ident, + position: Int, + name: Option[String], + pageCount: Option[Int] + ) + case class ListItemWithTags( + item: ListItem, + tags: List[RTag], + attachments: List[AttachmentLight] + ) /** Same as `findItems` but resolves the tags for each item. Note that * this is implemented by running an additional query per item. @@ -476,8 +486,29 @@ object QItem { item <- search tagItems <- Stream.eval(RTagItem.findByItem(item.id)) tags <- Stream.eval(tagItems.traverse(ti => findTag(resolvedTags, ti))) + attachs <- Stream.eval(findAttachmentLight(item.id)) ftags = tags.flatten.filter(t => t.collective == collective) - } yield ListItemWithTags(item, ftags.toList.sortBy(_.name)) + } yield ListItemWithTags( + item, + ftags.toList.sortBy(_.name), + attachs.sortBy(_.position) + ) + } + + private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { + val aId = RAttachment.Columns.id.prefix("a") + val aItem = RAttachment.Columns.itemId.prefix("a") + val aPos = RAttachment.Columns.position.prefix("a") + val aName = RAttachment.Columns.name.prefix("a") + val mId = RAttachmentMeta.Columns.id.prefix("m") + val mPages = RAttachmentMeta.Columns.pages.prefix("m") + + val cols = Seq(aId, aPos, aName, mPages) + val join = RAttachment.table ++ + fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) + val cond = aItem.is(item) + + selectSimple(cols, join, cond).query[AttachmentLight].to[List] } def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = From 67e8994aecc3ca56d16f3d13365c3da3cae1d7a1 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 14:29:52 +0100 Subject: [PATCH 03/10] Use attachment preview urls This changes the preview urls to use the concrete attachment ids. This way browsers have it easier to switch the preview image when the attachment position is changed. --- modules/webapp/src/main/elm/Api.elm | 17 +++++++++++++++-- .../webapp/src/main/elm/Comp/ItemCardList.elm | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 7230be7e..601f092e 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -144,6 +144,7 @@ import Api.Model.InviteResult exposing (InviteResult) import Api.Model.ItemDetail exposing (ItemDetail) import Api.Model.ItemFtsSearch exposing (ItemFtsSearch) import Api.Model.ItemInsights exposing (ItemInsights) +import Api.Model.ItemLight exposing (ItemLight) import Api.Model.ItemLightList exposing (ItemLightList) import Api.Model.ItemProposals exposing (ItemProposals) import Api.Model.ItemSearch exposing (ItemSearch) @@ -1503,8 +1504,20 @@ deleteAllItems flags ids receive = --- Item -itemPreviewURL : String -> String -itemPreviewURL itemId = +itemPreviewURL : ItemLight -> String +itemPreviewURL item = + let + makeUrl a = + "/api/v1/sec/attachment/" ++ a.id ++ "/preview?withFallback=true" + in + List.sortBy .position item.attachments + |> List.head + |> Maybe.map makeUrl + |> Maybe.withDefault (itemBasePreviewURL item.id) + + +itemBasePreviewURL : String -> String +itemBasePreviewURL itemId = "/api/v1/sec/item/" ++ itemId ++ "/preview?withFallback=true" diff --git a/modules/webapp/src/main/elm/Comp/ItemCardList.elm b/modules/webapp/src/main/elm/Comp/ItemCardList.elm index b9595b67..2b6fd4fc 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCardList.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCardList.elm @@ -238,7 +238,7 @@ viewItem cfg settings item = div [ class "image" ] [ img [ class "preview-image" - , src (Api.itemPreviewURL item.id) + , src (Api.itemPreviewURL item) , Data.UiSettings.cardPreviewSize settings ] [] From 848c245db6d0da1505a80907145166eb38b11334 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 14:36:58 +0100 Subject: [PATCH 04/10] Change the card link to only use the main content The card is no longer a link itself. The main target is moved to be the content (the area containing the title and tags). This is in preparation of upcoming changes: if the whole card is a link, it cannot contain other links, due to a restriction by html. Later a card may have more links to provide. --- modules/webapp/src/main/elm/Comp/ItemCardList.elm | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/ItemCardList.elm b/modules/webapp/src/main/elm/Comp/ItemCardList.elm index 2b6fd4fc..675847b3 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCardList.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCardList.elm @@ -219,15 +219,13 @@ viewItem cfg settings item = Data.ItemSelection.Active ids -> onClick (ToggleSelectItem ids item.id) in - a + div ([ classList [ ( "ui fluid card", True ) , ( cardColor, True ) , ( "current", cfg.current == Just item.id ) ] , id item.id - , href "#" - , cardAction ] ++ DD.draggable ItemDDMsg item.id ) @@ -243,7 +241,11 @@ viewItem cfg settings item = ] [] ] - , div [ class "content" ] + , a + [ class "content" + , href "#" + , cardAction + ] [ case cfg.selection of Data.ItemSelection.Active ids -> div [ class "header" ] From 7a14b05ea70a949b11ae7ae3345097ddd7ec09fb Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 17:04:56 +0100 Subject: [PATCH 05/10] Enhance item card displaying current file and number of pages --- modules/webapp/src/main/elm/Api.elm | 16 +- modules/webapp/src/main/elm/Comp/ItemCard.elm | 442 ++++++++++++++++++ .../webapp/src/main/elm/Comp/ItemCardList.elm | 332 ++----------- modules/webapp/src/main/webjar/docspell.css | 11 +- 4 files changed, 490 insertions(+), 311 deletions(-) create mode 100644 modules/webapp/src/main/elm/Comp/ItemCard.elm diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 601f092e..c9935f49 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -6,6 +6,7 @@ module Api exposing , addMember , addTag , addTagsMultiple + , attachmentPreviewURL , cancelJob , changeFolderName , changePassword @@ -58,9 +59,9 @@ module Api exposing , getTagCloud , getTags , getUsers + , itemBasePreviewURL , itemDetail , itemIndexSearch - , itemPreviewURL , itemSearch , login , loginSession @@ -1504,16 +1505,9 @@ deleteAllItems flags ids receive = --- Item -itemPreviewURL : ItemLight -> String -itemPreviewURL item = - let - makeUrl a = - "/api/v1/sec/attachment/" ++ a.id ++ "/preview?withFallback=true" - in - List.sortBy .position item.attachments - |> List.head - |> Maybe.map makeUrl - |> Maybe.withDefault (itemBasePreviewURL item.id) +attachmentPreviewURL : String -> String +attachmentPreviewURL id = + "/api/v1/sec/attachment/" ++ id ++ "/preview?withFallback=true" itemBasePreviewURL : String -> String diff --git a/modules/webapp/src/main/elm/Comp/ItemCard.elm b/modules/webapp/src/main/elm/Comp/ItemCard.elm new file mode 100644 index 00000000..7c48245a --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ItemCard.elm @@ -0,0 +1,442 @@ +module Comp.ItemCard exposing (..) + +import Api +import Api.Model.AttachmentLight exposing (AttachmentLight) +import Api.Model.HighlightEntry exposing (HighlightEntry) +import Api.Model.ItemLight exposing (ItemLight) +import Data.Direction +import Data.Fields +import Data.Icons as Icons +import Data.ItemSelection exposing (ItemSelection) +import Data.UiSettings exposing (UiSettings) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Markdown +import Page exposing (Page(..)) +import Set exposing (Set) +import Util.Html +import Util.ItemDragDrop as DD +import Util.List +import Util.Maybe +import Util.String +import Util.Time + + +type alias Model = + { previewAttach : Maybe AttachmentLight + } + + +type Msg + = CyclePreview ItemLight + | ToggleSelectItem (Set String) String + | ItemDDMsg DD.Msg + + +type alias ViewConfig = + { selection : ItemSelection + , extraClasses : String + } + + +type alias UpdateResult = + { model : Model + , dragModel : DD.Model + , selection : ItemSelection + } + + +init : Model +init = + { previewAttach = Nothing + } + + +currentAttachment : Model -> ItemLight -> Maybe AttachmentLight +currentAttachment model item = + Util.Maybe.or + [ model.previewAttach + , List.head item.attachments + ] + + +currentPosition : Model -> ItemLight -> Int +currentPosition model item = + let + filter cur el = + cur.id == el.id + in + case model.previewAttach of + Just a -> + case Util.List.findIndexed (filter a) item.attachments of + Just ( _, n ) -> + n + 1 + + Nothing -> + 1 + + Nothing -> + 1 + + +update : DD.Model -> Msg -> Model -> UpdateResult +update ddm msg model = + case msg of + ItemDDMsg lm -> + let + ddd = + DD.update lm ddm + in + UpdateResult model ddd.model Data.ItemSelection.Inactive + + ToggleSelectItem ids id -> + let + newSet = + if Set.member id ids then + Set.remove id ids + + else + Set.insert id ids + in + UpdateResult model ddm (Data.ItemSelection.Active newSet) + + CyclePreview item -> + let + mainAttach = + currentAttachment model item + + next = + Util.List.findNext (\e -> Just e.id == Maybe.map .id mainAttach) item.attachments + in + UpdateResult { model | previewAttach = next } + ddm + Data.ItemSelection.Inactive + + +view : ViewConfig -> UiSettings -> Model -> ItemLight -> Html Msg +view cfg settings model item = + let + dirIcon = + i [ class (Data.Direction.iconFromMaybe item.direction) ] [] + + corr = + List.filterMap identity [ item.corrOrg, item.corrPerson ] + |> List.map .name + |> List.intersperse ", " + |> String.concat + + conc = + List.filterMap identity [ item.concPerson, item.concEquip ] + |> List.map .name + |> List.intersperse ", " + |> String.concat + + folder = + Maybe.map .name item.folder + |> Maybe.withDefault "" + + dueDate = + Maybe.map Util.Time.formatDateShort item.dueDate + |> Maybe.withDefault "" + + isConfirmed = + item.state /= "created" + + cardColor = + if isSelected cfg item.id then + "purple" + + else if not isConfirmed then + "blue" + + else + "" + + fieldHidden f = + Data.UiSettings.fieldHidden settings f + + cardAction = + case cfg.selection of + Data.ItemSelection.Inactive -> + Page.href (ItemDetailPage item.id) + + Data.ItemSelection.Active ids -> + onClick (ToggleSelectItem ids item.id) + + mainAttach = + currentAttachment model item + + previewUrl = + Maybe.map .id mainAttach + |> Maybe.map Api.attachmentPreviewURL + |> Maybe.withDefault (Api.itemBasePreviewURL item.id) + + pageCount = + Maybe.andThen .pageCount mainAttach + |> Maybe.withDefault 0 + + pageCountLabel = + div + [ classList + [ ( "card-attachment-nav", True ) + , ( "invisible", pageCount == 0 ) + ] + ] + [ if item.fileCount == 1 then + div + [ class "ui secondary basic mini label" + , title "Number of pages" + ] + [ text "p." + , text (String.fromInt pageCount) + ] + + else + div [ class "ui left labeled mini button" ] + [ div [ class "ui basic right pointing mini label" ] + [ currentPosition model item + |> String.fromInt + |> text + , text "/" + , text (String.fromInt item.fileCount) + , text " p." + , text (String.fromInt pageCount) + ] + , a + [ class "ui mini icon secondary button" + , href "#" + , onClick (CyclePreview item) + ] + [ i [ class "arrow right icon" ] [] + ] + ] + ] + in + div + ([ classList + [ ( "ui fluid card", True ) + , ( cardColor, True ) + , ( cfg.extraClasses, True ) + ] + , id item.id + ] + ++ DD.draggable ItemDDMsg item.id + ) + [ if fieldHidden Data.Fields.PreviewImage then + span [ class "invisible" ] [] + + else + div [ class "image" ] + [ img + [ class "preview-image" + , src previewUrl + , Data.UiSettings.cardPreviewSize settings + ] + [] + , pageCountLabel + ] + , a + [ class "link content" + , href "#" + , cardAction + ] + [ case cfg.selection of + Data.ItemSelection.Active ids -> + div [ class "header" ] + [ Util.Html.checkbox (Set.member item.id ids) + , dirIcon + , Util.String.underscoreToSpace item.name + |> text + ] + + Data.ItemSelection.Inactive -> + if fieldHidden Data.Fields.Direction then + div [ class "header" ] + [ Util.String.underscoreToSpace item.name |> text + ] + + else + div + [ class "header" + , Data.Direction.labelFromMaybe item.direction + |> title + ] + [ dirIcon + , Util.String.underscoreToSpace item.name + |> text + ] + , div + [ classList + [ ( "ui right corner label", True ) + , ( cardColor, True ) + , ( "invisible", isConfirmed ) + ] + , title "New" + ] + [ i [ class "exclamation icon" ] [] + ] + , div + [ classList + [ ( "meta", True ) + , ( "invisible hidden", fieldHidden Data.Fields.Date ) + ] + ] + [ Util.Time.formatDate item.date |> text + ] + , div [ class "meta description" ] + [ div + [ classList + [ ( "ui right floated tiny labels", True ) + , ( "invisible hidden", item.tags == [] || fieldHidden Data.Fields.Tag ) + ] + ] + (List.map + (\tag -> + div + [ classList + [ ( "ui basic label", True ) + , ( Data.UiSettings.tagColorString tag settings, True ) + ] + ] + [ text tag.name ] + ) + item.tags + ) + ] + ] + , div + [ classList + [ ( "content", True ) + , ( "invisible hidden" + , settings.itemSearchNoteLength + <= 0 + || Util.String.isNothingOrBlank item.notes + ) + ] + ] + [ span [ class "small-info" ] + [ Maybe.withDefault "" item.notes + |> Util.String.ellipsis settings.itemSearchNoteLength + |> text + ] + ] + , div [ class "content" ] + [ div [ class "ui horizontal list" ] + [ div + [ classList + [ ( "item", True ) + , ( "invisible hidden" + , fieldHidden Data.Fields.CorrOrg + && fieldHidden Data.Fields.CorrPerson + ) + ] + , title "Correspondent" + ] + [ Icons.correspondentIcon "" + , text " " + , Util.String.withDefault "-" corr |> text + ] + , div + [ classList + [ ( "item", True ) + , ( "invisible hidden" + , fieldHidden Data.Fields.ConcPerson + && fieldHidden Data.Fields.ConcEquip + ) + ] + , title "Concerning" + ] + [ Icons.concernedIcon + , text " " + , Util.String.withDefault "-" conc |> text + ] + , div + [ classList + [ ( "item", True ) + , ( "invisible hidden", fieldHidden Data.Fields.Folder ) + ] + , title "Folder" + ] + [ Icons.folderIcon "" + , text " " + , Util.String.withDefault "-" folder |> text + ] + ] + , div [ class "right floated meta" ] + [ div [ class "ui horizontal list" ] + [ div + [ class "item" + , title "Source" + ] + [ text item.source + ] + , div + [ classList + [ ( "item", True ) + , ( "invisible hidden" + , item.dueDate + == Nothing + || fieldHidden Data.Fields.DueDate + ) + ] + , title ("Due on " ++ dueDate) + ] + [ div + [ class "ui basic grey label" + ] + [ Icons.dueDateIcon "" + , text (" " ++ dueDate) + ] + ] + ] + ] + ] + , div + [ classList + [ ( "content search-highlight", True ) + , ( "invisible hidden", item.highlighting == [] ) + ] + ] + [ div [ class "ui list" ] + (List.map renderHighlightEntry item.highlighting) + ] + ] + + +renderHighlightEntry : HighlightEntry -> Html Msg +renderHighlightEntry entry = + let + stripWhitespace str = + String.trim str + |> String.replace "```" "" + |> String.replace "\t" " " + |> String.replace "\n\n" "\n" + |> String.lines + |> List.map String.trim + |> String.join "\n" + in + div [ class "item" ] + [ div [ class "content" ] + (div [ class "header" ] + [ i [ class "caret right icon" ] [] + , text (entry.name ++ ":") + ] + :: List.map + (\str -> + Markdown.toHtml [ class "description" ] <| + (stripWhitespace str ++ "…") + ) + entry.lines + ) + ] + + +isSelected : ViewConfig -> String -> Bool +isSelected cfg id = + case cfg.selection of + Data.ItemSelection.Active ids -> + Set.member id ids + + Data.ItemSelection.Inactive -> + False diff --git a/modules/webapp/src/main/elm/Comp/ItemCardList.elm b/modules/webapp/src/main/elm/Comp/ItemCardList.elm index 675847b3..24b942bb 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCardList.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCardList.elm @@ -10,46 +10,38 @@ module Comp.ItemCardList exposing , view ) -import Api -import Api.Model.HighlightEntry exposing (HighlightEntry) import Api.Model.ItemLight exposing (ItemLight) import Api.Model.ItemLightGroup exposing (ItemLightGroup) import Api.Model.ItemLightList exposing (ItemLightList) -import Data.Direction -import Data.Fields +import Comp.ItemCard import Data.Flags exposing (Flags) -import Data.Icons as Icons import Data.ItemSelection exposing (ItemSelection) import Data.Items import Data.UiSettings exposing (UiSettings) +import Dict exposing (Dict) import Html exposing (..) import Html.Attributes exposing (..) -import Html.Events exposing (onClick) -import Markdown import Page exposing (Page(..)) -import Set exposing (Set) -import Util.Html import Util.ItemDragDrop as DD import Util.List -import Util.String -import Util.Time type alias Model = { results : ItemLightList + , itemCards : Dict String Comp.ItemCard.Model } type Msg = SetResults ItemLightList | AddResults ItemLightList - | ItemDDMsg DD.Msg - | ToggleSelectItem (Set String) String + | ItemCardMsg ItemLight Comp.ItemCard.Msg init : Model init = { results = Api.Model.ItemLightList.empty + , itemCards = Dict.empty } @@ -112,23 +104,22 @@ updateDrag dm _ msg model = in UpdateResult newModel Cmd.none dm Data.ItemSelection.Inactive - ItemDDMsg lm -> + ItemCardMsg item lm -> let - ddd = - DD.update lm dm - in - UpdateResult model Cmd.none ddd.model Data.ItemSelection.Inactive + cardModel = + Dict.get item.id model.itemCards + |> Maybe.withDefault Comp.ItemCard.init - ToggleSelectItem ids id -> - let - newSet = - if Set.member id ids then - Set.remove id ids + result = + Comp.ItemCard.update dm lm cardModel - else - Set.insert id ids + cards = + Dict.insert item.id result.model model.itemCards in - UpdateResult model Cmd.none dm (Data.ItemSelection.Active newSet) + UpdateResult { model | itemCards = cards } + Cmd.none + result.dragModel + result.selection @@ -141,297 +132,42 @@ type alias ViewConfig = } -isSelected : ViewConfig -> String -> Bool -isSelected cfg id = - case cfg.selection of - Data.ItemSelection.Active ids -> - Set.member id ids - - Data.ItemSelection.Inactive -> - False - - view : ViewConfig -> UiSettings -> Model -> Html Msg view cfg settings model = div [ class "ui container" ] - (List.map (viewGroup cfg settings) model.results.groups) + (List.map (viewGroup model cfg settings) model.results.groups) -viewGroup : ViewConfig -> UiSettings -> ItemLightGroup -> Html Msg -viewGroup cfg settings group = +viewGroup : Model -> ViewConfig -> UiSettings -> ItemLightGroup -> Html Msg +viewGroup model cfg settings group = div [ class "item-group" ] [ div [ class "ui horizontal divider header item-list" ] [ i [ class "calendar alternate outline icon" ] [] , text group.name ] , div [ class "ui stackable three cards" ] - (List.map (viewItem cfg settings) group.items) + (List.map (viewItem model cfg settings) group.items) ] -viewItem : ViewConfig -> UiSettings -> ItemLight -> Html Msg -viewItem cfg settings item = +viewItem : Model -> ViewConfig -> UiSettings -> ItemLight -> Html Msg +viewItem model cfg settings item = let - dirIcon = - i [ class (Data.Direction.iconFromMaybe item.direction) ] [] - - corr = - List.filterMap identity [ item.corrOrg, item.corrPerson ] - |> List.map .name - |> List.intersperse ", " - |> String.concat - - conc = - List.filterMap identity [ item.concPerson, item.concEquip ] - |> List.map .name - |> List.intersperse ", " - |> String.concat - - folder = - Maybe.map .name item.folder - |> Maybe.withDefault "" - - dueDate = - Maybe.map Util.Time.formatDateShort item.dueDate - |> Maybe.withDefault "" - - isConfirmed = - item.state /= "created" - - cardColor = - if isSelected cfg item.id then - "purple" - - else if not isConfirmed then - "blue" + currentClass = + if cfg.current == Just item.id then + "current" else "" - fieldHidden f = - Data.UiSettings.fieldHidden settings f + vvcfg = + Comp.ItemCard.ViewConfig cfg.selection currentClass - cardAction = - case cfg.selection of - Data.ItemSelection.Inactive -> - Page.href (ItemDetailPage item.id) + cardModel = + Dict.get item.id model.itemCards + |> Maybe.withDefault Comp.ItemCard.init - Data.ItemSelection.Active ids -> - onClick (ToggleSelectItem ids item.id) + cardHtml = + Comp.ItemCard.view vvcfg settings cardModel item in - div - ([ classList - [ ( "ui fluid card", True ) - , ( cardColor, True ) - , ( "current", cfg.current == Just item.id ) - ] - , id item.id - ] - ++ DD.draggable ItemDDMsg item.id - ) - [ if fieldHidden Data.Fields.PreviewImage then - span [ class "invisible" ] [] - - else - div [ class "image" ] - [ img - [ class "preview-image" - , src (Api.itemPreviewURL item) - , Data.UiSettings.cardPreviewSize settings - ] - [] - ] - , a - [ class "content" - , href "#" - , cardAction - ] - [ case cfg.selection of - Data.ItemSelection.Active ids -> - div [ class "header" ] - [ Util.Html.checkbox (Set.member item.id ids) - , dirIcon - , Util.String.underscoreToSpace item.name - |> text - ] - - Data.ItemSelection.Inactive -> - if fieldHidden Data.Fields.Direction then - div [ class "header" ] - [ Util.String.underscoreToSpace item.name |> text - ] - - else - div - [ class "header" - , Data.Direction.labelFromMaybe item.direction - |> title - ] - [ dirIcon - , Util.String.underscoreToSpace item.name - |> text - ] - , div - [ classList - [ ( "ui right corner label", True ) - , ( cardColor, True ) - , ( "invisible", isConfirmed ) - ] - , title "New" - ] - [ i [ class "exclamation icon" ] [] - ] - , div - [ classList - [ ( "meta", True ) - , ( "invisible hidden", fieldHidden Data.Fields.Date ) - ] - ] - [ Util.Time.formatDate item.date |> text - ] - , div [ class "meta description" ] - [ div - [ classList - [ ( "ui right floated tiny labels", True ) - , ( "invisible hidden", item.tags == [] || fieldHidden Data.Fields.Tag ) - ] - ] - (List.map - (\tag -> - div - [ classList - [ ( "ui basic label", True ) - , ( Data.UiSettings.tagColorString tag settings, True ) - ] - ] - [ text tag.name ] - ) - item.tags - ) - ] - ] - , div - [ classList - [ ( "content", True ) - , ( "invisible hidden" - , settings.itemSearchNoteLength - <= 0 - || Util.String.isNothingOrBlank item.notes - ) - ] - ] - [ span [ class "small-info" ] - [ Maybe.withDefault "" item.notes - |> Util.String.ellipsis settings.itemSearchNoteLength - |> text - ] - ] - , div [ class "content" ] - [ div [ class "ui horizontal list" ] - [ div - [ classList - [ ( "item", True ) - , ( "invisible hidden" - , fieldHidden Data.Fields.CorrOrg - && fieldHidden Data.Fields.CorrPerson - ) - ] - , title "Correspondent" - ] - [ Icons.correspondentIcon "" - , text " " - , Util.String.withDefault "-" corr |> text - ] - , div - [ classList - [ ( "item", True ) - , ( "invisible hidden" - , fieldHidden Data.Fields.ConcPerson - && fieldHidden Data.Fields.ConcEquip - ) - ] - , title "Concerning" - ] - [ Icons.concernedIcon - , text " " - , Util.String.withDefault "-" conc |> text - ] - , div - [ classList - [ ( "item", True ) - , ( "invisible hidden", fieldHidden Data.Fields.Folder ) - ] - , title "Folder" - ] - [ Icons.folderIcon "" - , text " " - , Util.String.withDefault "-" folder |> text - ] - ] - , div [ class "right floated meta" ] - [ div [ class "ui horizontal list" ] - [ div - [ class "item" - , title "Source" - ] - [ text item.source - ] - , div - [ classList - [ ( "item", True ) - , ( "invisible hidden" - , item.dueDate - == Nothing - || fieldHidden Data.Fields.DueDate - ) - ] - , title ("Due on " ++ dueDate) - ] - [ div - [ class "ui basic grey label" - ] - [ Icons.dueDateIcon "" - , text (" " ++ dueDate) - ] - ] - ] - ] - ] - , div - [ classList - [ ( "content search-highlight", True ) - , ( "invisible hidden", item.highlighting == [] ) - ] - ] - [ div [ class "ui list" ] - (List.map renderHighlightEntry item.highlighting) - ] - ] - - -renderHighlightEntry : HighlightEntry -> Html Msg -renderHighlightEntry entry = - let - stripWhitespace str = - String.trim str - |> String.replace "```" "" - |> String.replace "\t" " " - |> String.replace "\n\n" "\n" - |> String.lines - |> List.map String.trim - |> String.join "\n" - in - div [ class "item" ] - [ div [ class "content" ] - (div [ class "header" ] - [ i [ class "caret right icon" ] [] - , text (entry.name ++ ":") - ] - :: List.map - (\str -> - Markdown.toHtml [ class "description" ] <| - (stripWhitespace str ++ "…") - ) - entry.lines - ) - ] + Html.map (ItemCardMsg item) cardHtml diff --git a/modules/webapp/src/main/webjar/docspell.css b/modules/webapp/src/main/webjar/docspell.css index 04ec1eff..58387cac 100644 --- a/modules/webapp/src/main/webjar/docspell.css +++ b/modules/webapp/src/main/webjar/docspell.css @@ -92,7 +92,15 @@ background: floralwhite; padding: 0.8em; } - +.default-layout .ui.card .link.content:hover { + box-shadow: 0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15); +} +.default-layout .image .card-attachment-nav { + position: absolute; + bottom: 2px; + right: 2px; + z-index: 10; +} .default-layout img.preview-image { margin-left: auto; margin-right: auto; @@ -119,7 +127,6 @@ background: rgba(220, 255, 71, 0.6); } .default-layout .ui.cards .ui.card.current { - /* semantic-ui purple */ box-shadow: 0 0 6px rgba(0,0,0,0.55); } From 89646ef3f6309058a1db7dd6cffbd4e67316c68d Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 17:06:44 +0100 Subject: [PATCH 06/10] Hide number of pages, if item is only one file with one page --- modules/webapp/src/main/elm/Comp/ItemCard.elm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/webapp/src/main/elm/Comp/ItemCard.elm b/modules/webapp/src/main/elm/Comp/ItemCard.elm index 7c48245a..21563564 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCard.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCard.elm @@ -180,7 +180,7 @@ view cfg settings model item = div [ classList [ ( "card-attachment-nav", True ) - , ( "invisible", pageCount == 0 ) + , ( "invisible", pageCount == 0 || (item.fileCount == 1 && pageCount == 1) ) ] ] [ if item.fileCount == 1 then From de00b46e5d5158351e6acfad6442ef4ed9b300a3 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 17:12:06 +0100 Subject: [PATCH 07/10] Move searchbar to the right --- modules/webapp/src/main/elm/Page/Home/View.elm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index bfc10bf7..5a506ccc 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -307,7 +307,7 @@ viewSearchBar flags model = ] [ i [ class "filter icon" ] [] ] - , div [ class "item" ] + , div [ class "right fitted item" ] [ div [ class "ui left icon right action input" ] [ i [ classList From 29455d638c0a0919ddf2cff183f69b33b528e050 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 20:34:34 +0100 Subject: [PATCH 08/10] Add startup task to find page counts of existing files --- .../scala/docspell/backend/JobFactory.scala | 20 +++++ .../docspell/common/DocspellSystem.scala | 9 ++- .../docspell/common/MakePageCountArgs.scala | 24 ++++++ .../scala/docspell/joex/JoexAppImpl.scala | 18 ++++- .../joex/pagecount/AllPageCountTask.scala | 75 +++++++++++++++++++ .../joex/pagecount/PageCountTask.scala | 55 ++++++++++++++ .../joex/process/AttachmentPageCount.scala | 27 +++++-- .../restserver/conv/Conversions.scala | 2 +- .../docspell/store/records/RAttachment.scala | 15 ++++ 9 files changed, 234 insertions(+), 11 deletions(-) create mode 100644 modules/common/src/main/scala/docspell/common/MakePageCountArgs.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index fdb0d860..56ac2566 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -8,6 +8,26 @@ import docspell.store.records.RJob object JobFactory { + def makePageCount[F[_]: Sync]( + args: MakePageCountArgs, + account: Option[AccountId] + ): F[RJob] = + for { + id <- Ident.randomId[F] + now <- Timestamp.current[F] + job = RJob.newJob( + id, + MakePageCountArgs.taskName, + account.map(_.collective).getOrElse(DocspellSystem.taskGroup), + args, + s"Find page-count metadata for ${args.attachment.id}", + now, + account.map(_.user).getOrElse(DocspellSystem.user), + Priority.Low, + Some(MakePageCountArgs.taskName / args.attachment) + ) + } yield job + def makePreview[F[_]: Sync]( args: MakePreviewArgs, account: Option[AccountId] diff --git a/modules/common/src/main/scala/docspell/common/DocspellSystem.scala b/modules/common/src/main/scala/docspell/common/DocspellSystem.scala index ad410281..def2ade2 100644 --- a/modules/common/src/main/scala/docspell/common/DocspellSystem.scala +++ b/modules/common/src/main/scala/docspell/common/DocspellSystem.scala @@ -2,8 +2,9 @@ package docspell.common object DocspellSystem { - val user = Ident.unsafe("docspell-system") - val taskGroup = user - val migrationTaskTracker = Ident.unsafe("full-text-index-tracker") - val allPreviewTaskTracker = Ident.unsafe("generate-all-previews") + val user = Ident.unsafe("docspell-system") + val taskGroup = user + val migrationTaskTracker = Ident.unsafe("full-text-index-tracker") + val allPreviewTaskTracker = Ident.unsafe("generate-all-previews") + val allPageCountTaskTracker = Ident.unsafe("all-page-count-tracker") } diff --git a/modules/common/src/main/scala/docspell/common/MakePageCountArgs.scala b/modules/common/src/main/scala/docspell/common/MakePageCountArgs.scala new file mode 100644 index 00000000..ed955213 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/MakePageCountArgs.scala @@ -0,0 +1,24 @@ +package docspell.common + +import io.circe.generic.semiauto._ +import io.circe.{Decoder, Encoder} + +/** Arguments for the `MakePageCountTask` that reads the number of + * pages for an attachment and stores it into the meta data of the + * attachment. + */ +case class MakePageCountArgs( + attachment: Ident +) + +object MakePageCountArgs { + + val taskName = Ident.unsafe("make-page-count") + + implicit val jsonEncoder: Encoder[MakePageCountArgs] = + deriveEncoder[MakePageCountArgs] + + implicit val jsonDecoder: Decoder[MakePageCountArgs] = + deriveDecoder[MakePageCountArgs] + +} diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 2b9b96c5..51fed2bc 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -16,6 +16,7 @@ import docspell.joex.fts.{MigrationTask, ReIndexTask} import docspell.joex.hk._ import docspell.joex.learn.LearnClassifierTask import docspell.joex.notify._ +import docspell.joex.pagecount._ import docspell.joex.pdfconv.ConvertAllPdfTask import docspell.joex.pdfconv.PdfConvTask import docspell.joex.preview._ @@ -72,7 +73,8 @@ final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer]( MigrationTask.job.flatMap(queue.insertIfNew) *> AllPreviewsTask .job(MakePreviewArgs.StoreMode.WhenMissing, None) - .flatMap(queue.insertIfNew) + .flatMap(queue.insertIfNew) *> + AllPageCountTask.job.flatMap(queue.insertIfNew) } object JoexAppImpl { @@ -185,6 +187,20 @@ object JoexAppImpl { AllPreviewsTask.onCancel[F] ) ) + .withTask( + JobTask.json( + MakePageCountArgs.taskName, + MakePageCountTask[F](), + MakePageCountTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + AllPageCountTask.taskName, + AllPageCountTask[F](queue, joex), + AllPageCountTask.onCancel[F] + ) + ) .resource psch <- PeriodicScheduler.create( cfg.periodicScheduler, diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala new file mode 100644 index 00000000..43a93146 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -0,0 +1,75 @@ +package docspell.joex.pagecount + +import cats.effect._ +import cats.implicits._ +import fs2.{Chunk, Stream} + +import docspell.backend.JobFactory +import docspell.backend.ops.OJoex +import docspell.common._ +import docspell.joex.scheduler.Context +import docspell.joex.scheduler.Task +import docspell.store.queue.JobQueue +import docspell.store.records.RAttachment +import docspell.store.records.RJob + +object AllPageCountTask { + + val taskName = Ident.unsafe("all-page-count") + type Args = Unit + + def apply[F[_]: Sync](queue: JobQueue[F], joex: OJoex[F]): Task[F, Args, Unit] = + Task { ctx => + for { + _ <- ctx.logger.info("Generating previews for attachments") + n <- submitConversionJobs(ctx, queue) + _ <- ctx.logger.info(s"Submitted $n jobs") + _ <- joex.notifyAllNodes + } yield () + } + + def onCancel[F[_]: Sync]: Task[F, Args, Unit] = + Task.log(_.warn("Cancelling all-previews task")) + + def submitConversionJobs[F[_]: Sync]( + ctx: Context[F, Args], + queue: JobQueue[F] + ): F[Int] = + ctx.store + .transact(findAttachments) + .chunks + .flatMap(createJobs[F]) + .chunks + .evalMap(jobs => queue.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) + .evalTap(n => ctx.logger.debug(s"Submitted $n jobs …")) + .compile + .foldMonoid + + private def findAttachments[F[_]] = + RAttachment.findAllWithoutPageCount(50) + + private def createJobs[F[_]: Sync](ras: Chunk[RAttachment]): Stream[F, RJob] = { + def mkJob(ra: RAttachment): F[RJob] = + JobFactory.makePageCount(MakePageCountArgs(ra.id), None) + + val jobs = ras.traverse(mkJob) + Stream.evalUnChunk(jobs) + } + + def job[F[_]: Sync]: F[RJob] = + for { + id <- Ident.randomId[F] + now <- Timestamp.current[F] + } yield RJob.newJob( + id, + AllPageCountTask.taskName, + DocspellSystem.taskGroup, + (), + "Create all page-counts", + now, + DocspellSystem.taskGroup, + Priority.Low, + Some(DocspellSystem.allPageCountTaskTracker) + ) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala new file mode 100644 index 00000000..7e97186c --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala @@ -0,0 +1,55 @@ +package docspell.joex.pagecount + +import cats.effect._ +import cats.implicits._ + +import docspell.common._ +import docspell.joex.process.AttachmentPageCount +import docspell.joex.scheduler.Context +import docspell.joex.scheduler.Task +import docspell.store.records.RAttachment +import docspell.store.records.RAttachmentMeta + +object MakePageCountTask { + + type Args = MakePageCountArgs + + def apply[F[_]: Sync](): Task[F, Args, Unit] = + Task { ctx => + for { + exists <- pageCountExists(ctx) + _ <- + if (exists) + ctx.logger.info( + s"PageCount already exists for attachment ${ctx.args.attachment}. Skipping." + ) + else + ctx.logger.info( + s"Reading page-count for attachment ${ctx.args.attachment}" + ) *> generatePageCount(ctx) + } yield () + } + + def onCancel[F[_]: Sync]: Task[F, Args, Unit] = + Task.log(_.warn("Cancelling make-page-count task")) + + private def generatePageCount[F[_]: Sync]( + ctx: Context[F, Args] + ): F[Unit] = + for { + ra <- ctx.store.transact(RAttachment.findById(ctx.args.attachment)) + _ <- ra + .map(AttachmentPageCount.createPageCount(ctx)) + .getOrElse( + ctx.logger.warn(s"No attachment found with id: ${ctx.args.attachment}") + ) + } yield () + + private def pageCountExists[F[_]: Sync](ctx: Context[F, Args]): F[Boolean] = + ctx.store.transact( + RAttachmentMeta + .findById(ctx.args.attachment) + .map(_.flatMap(_.pages).exists(_ > 0)) + ) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala index c1dbe7e4..f3cf7b0e 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala @@ -50,14 +50,16 @@ object AttachmentPageCount { case MimeType.PdfMatch(_) => PdfboxExtract.getMetaData(loadFile(ctx)(ra)).flatMap { case Right(md) => - updatePageCount(ctx, md, ra).map(_.some) + ctx.logger.debug(s"Found number of pages: ${md.pageCount}") *> + updatePageCount(ctx, md, ra).map(_.some) case Left(ex) => ctx.logger.warn(s"Error obtaining pages count: ${ex.getMessage}") *> (None: Option[PdfMetaData]).pure[F] } - case _ => - (None: Option[PdfMetaData]).pure[F] + case mt => + ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot get page count.") *> + (None: Option[PdfMetaData]).pure[F] } private def updatePageCount[F[_]: Sync]( @@ -65,8 +67,23 @@ object AttachmentPageCount { md: PdfMetaData, ra: RAttachment ): F[PdfMetaData] = - ctx.store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some)) *> md - .pure[F] + for { + _ <- ctx.logger.debug( + s"Update attachment ${ra.id.id} with page count ${md.pageCount.some}" + ) + n <- ctx.store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some)) + m <- + if (n == 0) + ctx.logger.warn( + s"No attachmentmeta record exists for ${ra.id.id}. Creating new." + ) *> ctx.store.transact( + RAttachmentMeta.insert( + RAttachmentMeta(ra.id, None, Nil, MetaProposalList.empty, md.pageCount.some) + ) + ) + else 0.pure[F] + _ <- ctx.logger.debug(s"Stored page count (${n + m}).") + } yield md def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index fc0ccb75..aba61555 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -15,6 +15,7 @@ import docspell.common.syntax.all._ import docspell.ftsclient.FtsResult import docspell.restapi.model._ import docspell.restserver.conv.Conversions._ +import docspell.store.queries.QItem import docspell.store.records._ import docspell.store.{AddResult, UpdateResult} @@ -22,7 +23,6 @@ import bitpeace.FileMeta import org.http4s.headers.`Content-Type` import org.http4s.multipart.Multipart import org.log4s.Logger -import docspell.store.queries.QItem trait Conversions { diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index fa1453b6..b23c4146 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -255,6 +255,21 @@ object RAttachment { } } + def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = { + val aId = Columns.id.prefix("a") + val mId = RAttachmentMeta.Columns.id.prefix("m") + val mPages = RAttachmentMeta.Columns.pages.prefix("m") + + val cols = all.map(_.prefix("a")) + val join = table ++ fr"a LEFT OUTER JOIN" ++ + RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) + val cond = mPages.isNull + + selectSimple(cols, join, cond) + .query[RAttachment] + .streamWithChunkSize(chunkSize) + } + def findWithoutPreview( coll: Option[Ident], chunkSize: Int From 10305bc82d8d2804b026bea162a9288f24367807 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 21:16:53 +0100 Subject: [PATCH 09/10] Minor improvements --- .../joex/pagecount/PageCountTask.scala | 4 ++-- .../joex/preview/AllPreviewsTask.scala | 19 +++++---------- .../docspell/store/records/RAttachment.scala | 24 ++++++++++--------- .../store/records/RAttachmentMeta.scala | 6 +++++ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala index 7e97186c..d69d4fe3 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala @@ -48,8 +48,8 @@ object MakePageCountTask { private def pageCountExists[F[_]: Sync](ctx: Context[F, Args]): F[Boolean] = ctx.store.transact( RAttachmentMeta - .findById(ctx.args.attachment) - .map(_.flatMap(_.pages).exists(_ > 0)) + .findPageCountById(ctx.args.attachment) + .map(_.exists(_ > 0)) ) } diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index 70d87fdb..9efed3a1 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -56,22 +56,15 @@ object AllPreviewsTask { private def createJobs[F[_]: Sync]( ctx: Context[F, Args] )(ras: Chunk[RAttachment]): Stream[F, RJob] = { - val collectiveOrSystem = ctx.args.collective.getOrElse(DocspellSystem.taskGroup) + val collectiveOrSystem = { + val cid = ctx.args.collective.getOrElse(DocspellSystem.taskGroup) + AccountId(cid, DocspellSystem.user) + } def mkJob(ra: RAttachment): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, - MakePreviewArgs.taskName, - collectiveOrSystem, + JobFactory.makePreview( MakePreviewArgs(ra.id, ctx.args.storeMode), - s"Create preview ${ra.id.id}/${ra.name.getOrElse("-")}", - now, - collectiveOrSystem, - Priority.Low, - Some(MakePreviewArgs.taskName / ra.id) + collectiveOrSystem.some ) val jobs = ras.traverse(mkJob) diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index b23c4146..e9d5d935 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -256,16 +256,17 @@ object RAttachment { } def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = { - val aId = Columns.id.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mPages = RAttachmentMeta.Columns.pages.prefix("m") + val aId = Columns.id.prefix("a") + val aCreated = Columns.created.prefix("a") + val mId = RAttachmentMeta.Columns.id.prefix("m") + val mPages = RAttachmentMeta.Columns.pages.prefix("m") val cols = all.map(_.prefix("a")) val join = table ++ fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) val cond = mPages.isNull - selectSimple(cols, join, cond) + (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) .query[RAttachment] .streamWithChunkSize(chunkSize) } @@ -274,11 +275,12 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { - val aId = Columns.id.prefix("a") - val aItem = Columns.itemId.prefix("a") - val pId = RAttachmentPreview.Columns.id.prefix("p") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") + val aId = Columns.id.prefix("a") + val aItem = Columns.itemId.prefix("a") + val aCreated = Columns.created.prefix("a") + val pId = RAttachmentPreview.Columns.id.prefix("p") + val iId = RItem.Columns.id.prefix("i") + val iColl = RItem.Columns.cid.prefix("i") val cols = all.map(_.prefix("a")) val baseJoin = @@ -292,11 +294,11 @@ object RAttachment { case Some(cid) => val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) val cond = and(baseCond ++ Seq(iColl.is(cid))) - selectSimple(cols, join, cond) + (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) .query[RAttachment] .streamWithChunkSize(chunkSize) case None => - selectSimple(cols, baseJoin, and(baseCond)) + (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc)) .query[RAttachment] .streamWithChunkSize(chunkSize) } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 833bfeca..5fcd5b93 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -54,6 +54,12 @@ object RAttachmentMeta { def findById(attachId: Ident): ConnectionIO[Option[RAttachmentMeta]] = selectSimple(all, table, id.is(attachId)).query[RAttachmentMeta].option + def findPageCountById(attachId: Ident): ConnectionIO[Option[Int]] = + selectSimple(Seq(pages), table, id.is(attachId)) + .query[Option[Int]] + .option + .map(_.flatten) + def upsert(v: RAttachmentMeta): ConnectionIO[Int] = for { n0 <- update(v) From 976aa757106df1b3276129a3429cd1474bd6d502 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 9 Nov 2020 21:36:06 +0100 Subject: [PATCH 10/10] Move card size definition in css and fix height bug For very tall images (sometimes shopping receipts), the height must be restricted, too. --- modules/webapp/src/main/elm/Comp/ItemCard.elm | 7 +++-- .../webapp/src/main/elm/Data/UiSettings.elm | 11 ++----- modules/webapp/src/main/webjar/docspell.css | 29 ++++++++++++++++++- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/ItemCard.elm b/modules/webapp/src/main/elm/Comp/ItemCard.elm index 21563564..95e758cc 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCard.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCard.elm @@ -179,7 +179,7 @@ view cfg settings model item = pageCountLabel = div [ classList - [ ( "card-attachment-nav", True ) + [ ( "card-attachment-nav top", True ) , ( "invisible", pageCount == 0 || (item.fileCount == 1 && pageCount == 1) ) ] ] @@ -227,7 +227,10 @@ view cfg settings model item = span [ class "invisible" ] [] else - div [ class "image" ] + div + [ class "image ds-card-image" + , Data.UiSettings.cardPreviewSize settings + ] [ img [ class "preview-image" , src previewUrl diff --git a/modules/webapp/src/main/elm/Data/UiSettings.elm b/modules/webapp/src/main/elm/Data/UiSettings.elm index 8d955549..5a0bc13f 100644 --- a/modules/webapp/src/main/elm/Data/UiSettings.elm +++ b/modules/webapp/src/main/elm/Data/UiSettings.elm @@ -226,15 +226,8 @@ fieldHidden settings field = cardPreviewSize : UiSettings -> Attribute msg cardPreviewSize settings = - case settings.cardPreviewSize of - Data.BasicSize.Small -> - HA.style "max-width" "80px" - - Data.BasicSize.Medium -> - HA.style "max-width" "160px" - - Data.BasicSize.Large -> - HA.style "max-width" "none" + Data.BasicSize.asString settings.cardPreviewSize + |> HA.class diff --git a/modules/webapp/src/main/webjar/docspell.css b/modules/webapp/src/main/webjar/docspell.css index 58387cac..1fbaa215 100644 --- a/modules/webapp/src/main/webjar/docspell.css +++ b/modules/webapp/src/main/webjar/docspell.css @@ -92,19 +92,46 @@ background: floralwhite; padding: 0.8em; } + .default-layout .ui.card .link.content:hover { box-shadow: 0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15); } .default-layout .image .card-attachment-nav { position: absolute; - bottom: 2px; right: 2px; z-index: 10; } +.default-layout .image .card-attachment-nav.bottom { + bottom: 2px; +} +.default-layout .image .card-attachment-nav.top { + top: 2px; +} +.default-layout .image.ds-card-image { + overflow: auto; +} +.default-layout .image.ds-card-image.small { + max-height: 120px; +} +.default-layout .image.ds-card-image.medium { + max-height: 240px; +} +.default-layout .image.ds-card-image.large { + max-height: 600px; +} .default-layout img.preview-image { margin-left: auto; margin-right: auto; } +.default-layout img.preview-image.small { + max-width: 80px; +} +.default-layout img.preview-image.medium { + max-width: 160px; +} +.default-layout img.preview-image.large { + max-width: none; +} .default-layout .menu .item.active a.right-tab-icon-link { position: relative;