Merge pull request #431 from eikek/pages-metadata

Pages metadata
This commit is contained in:
mergify[bot] 2020-11-09 21:47:03 +00:00 committed by GitHub
commit 6a31336adb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 975 additions and 369 deletions

View File

@ -8,6 +8,26 @@ import docspell.store.records.RJob
object JobFactory {
def makePageCount[F[_]: Sync](
args: MakePageCountArgs,
account: Option[AccountId]
): F[RJob] =
for {
id <- Ident.randomId[F]
now <- Timestamp.current[F]
job = RJob.newJob(
id,
MakePageCountArgs.taskName,
account.map(_.collective).getOrElse(DocspellSystem.taskGroup),
args,
s"Find page-count metadata for ${args.attachment.id}",
now,
account.map(_.user).getOrElse(DocspellSystem.user),
Priority.Low,
Some(MakePageCountArgs.taskName / args.attachment)
)
} yield job
def makePreview[F[_]: Sync](
args: MakePreviewArgs,
account: Option[AccountId]

View File

@ -2,8 +2,9 @@ package docspell.common
object DocspellSystem {
val user = Ident.unsafe("docspell-system")
val taskGroup = user
val migrationTaskTracker = Ident.unsafe("full-text-index-tracker")
val allPreviewTaskTracker = Ident.unsafe("generate-all-previews")
val user = Ident.unsafe("docspell-system")
val taskGroup = user
val migrationTaskTracker = Ident.unsafe("full-text-index-tracker")
val allPreviewTaskTracker = Ident.unsafe("generate-all-previews")
val allPageCountTaskTracker = Ident.unsafe("all-page-count-tracker")
}

View File

@ -0,0 +1,24 @@
package docspell.common
import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder}
/** Arguments for the `MakePageCountTask` that reads the number of
* pages for an attachment and stores it into the meta data of the
* attachment.
*/
case class MakePageCountArgs(
attachment: Ident
)
object MakePageCountArgs {
val taskName = Ident.unsafe("make-page-count")
implicit val jsonEncoder: Encoder[MakePageCountArgs] =
deriveEncoder[MakePageCountArgs]
implicit val jsonDecoder: Decoder[MakePageCountArgs] =
deriveDecoder[MakePageCountArgs]
}

View File

@ -8,7 +8,8 @@ final case class PdfMetaData(
subject: Option[String],
keywords: Option[String],
creator: Option[String],
creationDate: Option[Timestamp]
creationDate: Option[Timestamp],
pageCount: Int
) {
def isEmpty: Boolean =
@ -17,7 +18,8 @@ final case class PdfMetaData(
subject.isEmpty &&
keywords.isEmpty &&
creator.isEmpty &&
creationDate.isEmpty
creationDate.isEmpty &&
pageCount <= 0
def nonEmpty: Boolean =
!isEmpty
@ -36,5 +38,5 @@ final case class PdfMetaData(
}
object PdfMetaData {
val empty = PdfMetaData(None, None, None, None, None, None)
val empty = PdfMetaData(None, None, None, None, None, None, 0)
}

View File

@ -20,21 +20,23 @@ object PdfboxExtract {
def getTextAndMetaData[F[_]: Sync](
data: Stream[F, Byte]
): F[Either[Throwable, (Text, Option[PdfMetaData])]] =
data.compile
.to(Array)
.map(bytes =>
Using(PDDocument.load(bytes)) { doc =>
for {
txt <- readText(doc)
md <- readMetaData(doc)
} yield (txt, Some(md).filter(_.nonEmpty))
}.toEither.flatten
)
PdfLoader
.withDocumentStream(data) { doc =>
(for {
txt <- readText(doc)
md <- readMetaData(doc)
} yield (txt, Some(md).filter(_.nonEmpty))).pure[F]
}
.attempt
.map(_.flatten)
def getText[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, Text]] =
data.compile
.to(Array)
.map(bytes => Using(PDDocument.load(bytes))(readText).toEither.flatten)
PdfLoader
.withDocumentStream(data) { doc =>
readText(doc).pure[F]
}
.attempt
.map(_.flatten)
def getText(is: InputStream): Either[Throwable, Text] =
Using(PDDocument.load(is))(readText).toEither.flatten
@ -51,9 +53,10 @@ object PdfboxExtract {
}.toEither
def getMetaData[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, PdfMetaData]] =
data.compile
.to(Array)
.map(bytes => Using(PDDocument.load(bytes))(readMetaData).toEither.flatten)
PdfLoader
.withDocumentStream(data)(doc => readMetaData(doc).pure[F])
.attempt
.map(_.flatten)
def getMetaData(is: InputStream): Either[Throwable, PdfMetaData] =
Using(PDDocument.load(is))(readMetaData).toEither.flatten
@ -73,7 +76,8 @@ object PdfboxExtract {
mkValue(info.getSubject),
mkValue(info.getKeywords),
mkValue(info.getCreator),
Option(info.getCreationDate).map(c => Timestamp(c.toInstant))
Option(info.getCreationDate).map(c => Timestamp(c.toInstant)),
doc.getNumberOfPages()
)
}.toEither
}

View File

@ -16,6 +16,7 @@ import docspell.joex.fts.{MigrationTask, ReIndexTask}
import docspell.joex.hk._
import docspell.joex.learn.LearnClassifierTask
import docspell.joex.notify._
import docspell.joex.pagecount._
import docspell.joex.pdfconv.ConvertAllPdfTask
import docspell.joex.pdfconv.PdfConvTask
import docspell.joex.preview._
@ -72,7 +73,8 @@ final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer](
MigrationTask.job.flatMap(queue.insertIfNew) *>
AllPreviewsTask
.job(MakePreviewArgs.StoreMode.WhenMissing, None)
.flatMap(queue.insertIfNew)
.flatMap(queue.insertIfNew) *>
AllPageCountTask.job.flatMap(queue.insertIfNew)
}
object JoexAppImpl {
@ -185,6 +187,20 @@ object JoexAppImpl {
AllPreviewsTask.onCancel[F]
)
)
.withTask(
JobTask.json(
MakePageCountArgs.taskName,
MakePageCountTask[F](),
MakePageCountTask.onCancel[F]
)
)
.withTask(
JobTask.json(
AllPageCountTask.taskName,
AllPageCountTask[F](queue, joex),
AllPageCountTask.onCancel[F]
)
)
.resource
psch <- PeriodicScheduler.create(
cfg.periodicScheduler,

View File

@ -0,0 +1,75 @@
package docspell.joex.pagecount
import cats.effect._
import cats.implicits._
import fs2.{Chunk, Stream}
import docspell.backend.JobFactory
import docspell.backend.ops.OJoex
import docspell.common._
import docspell.joex.scheduler.Context
import docspell.joex.scheduler.Task
import docspell.store.queue.JobQueue
import docspell.store.records.RAttachment
import docspell.store.records.RJob
object AllPageCountTask {
val taskName = Ident.unsafe("all-page-count")
type Args = Unit
def apply[F[_]: Sync](queue: JobQueue[F], joex: OJoex[F]): Task[F, Args, Unit] =
Task { ctx =>
for {
_ <- ctx.logger.info("Generating previews for attachments")
n <- submitConversionJobs(ctx, queue)
_ <- ctx.logger.info(s"Submitted $n jobs")
_ <- joex.notifyAllNodes
} yield ()
}
def onCancel[F[_]: Sync]: Task[F, Args, Unit] =
Task.log(_.warn("Cancelling all-previews task"))
def submitConversionJobs[F[_]: Sync](
ctx: Context[F, Args],
queue: JobQueue[F]
): F[Int] =
ctx.store
.transact(findAttachments)
.chunks
.flatMap(createJobs[F])
.chunks
.evalMap(jobs => queue.insertAllIfNew(jobs.toVector).map(_ => jobs.size))
.evalTap(n => ctx.logger.debug(s"Submitted $n jobs …"))
.compile
.foldMonoid
private def findAttachments[F[_]] =
RAttachment.findAllWithoutPageCount(50)
private def createJobs[F[_]: Sync](ras: Chunk[RAttachment]): Stream[F, RJob] = {
def mkJob(ra: RAttachment): F[RJob] =
JobFactory.makePageCount(MakePageCountArgs(ra.id), None)
val jobs = ras.traverse(mkJob)
Stream.evalUnChunk(jobs)
}
def job[F[_]: Sync]: F[RJob] =
for {
id <- Ident.randomId[F]
now <- Timestamp.current[F]
} yield RJob.newJob(
id,
AllPageCountTask.taskName,
DocspellSystem.taskGroup,
(),
"Create all page-counts",
now,
DocspellSystem.taskGroup,
Priority.Low,
Some(DocspellSystem.allPageCountTaskTracker)
)
}

View File

@ -0,0 +1,55 @@
package docspell.joex.pagecount
import cats.effect._
import cats.implicits._
import docspell.common._
import docspell.joex.process.AttachmentPageCount
import docspell.joex.scheduler.Context
import docspell.joex.scheduler.Task
import docspell.store.records.RAttachment
import docspell.store.records.RAttachmentMeta
object MakePageCountTask {
type Args = MakePageCountArgs
def apply[F[_]: Sync](): Task[F, Args, Unit] =
Task { ctx =>
for {
exists <- pageCountExists(ctx)
_ <-
if (exists)
ctx.logger.info(
s"PageCount already exists for attachment ${ctx.args.attachment}. Skipping."
)
else
ctx.logger.info(
s"Reading page-count for attachment ${ctx.args.attachment}"
) *> generatePageCount(ctx)
} yield ()
}
def onCancel[F[_]: Sync]: Task[F, Args, Unit] =
Task.log(_.warn("Cancelling make-page-count task"))
private def generatePageCount[F[_]: Sync](
ctx: Context[F, Args]
): F[Unit] =
for {
ra <- ctx.store.transact(RAttachment.findById(ctx.args.attachment))
_ <- ra
.map(AttachmentPageCount.createPageCount(ctx))
.getOrElse(
ctx.logger.warn(s"No attachment found with id: ${ctx.args.attachment}")
)
} yield ()
private def pageCountExists[F[_]: Sync](ctx: Context[F, Args]): F[Boolean] =
ctx.store.transact(
RAttachmentMeta
.findPageCountById(ctx.args.attachment)
.map(_.exists(_ > 0))
)
}

View File

@ -56,22 +56,15 @@ object AllPreviewsTask {
private def createJobs[F[_]: Sync](
ctx: Context[F, Args]
)(ras: Chunk[RAttachment]): Stream[F, RJob] = {
val collectiveOrSystem = ctx.args.collective.getOrElse(DocspellSystem.taskGroup)
val collectiveOrSystem = {
val cid = ctx.args.collective.getOrElse(DocspellSystem.taskGroup)
AccountId(cid, DocspellSystem.user)
}
def mkJob(ra: RAttachment): F[RJob] =
for {
id <- Ident.randomId[F]
now <- Timestamp.current[F]
} yield RJob.newJob(
id,
MakePreviewArgs.taskName,
collectiveOrSystem,
JobFactory.makePreview(
MakePreviewArgs(ra.id, ctx.args.storeMode),
s"Create preview ${ra.id.id}/${ra.name.getOrElse("-")}",
now,
collectiveOrSystem,
Priority.Low,
Some(MakePreviewArgs.taskName / ra.id)
collectiveOrSystem.some
)
val jobs = ras.traverse(mkJob)

View File

@ -0,0 +1,100 @@
package docspell.joex.process
import cats.Functor
import cats.data.OptionT
import cats.effect._
import cats.implicits._
import fs2.Stream
import docspell.common._
import docspell.extract.pdfbox.PdfMetaData
import docspell.extract.pdfbox.PdfboxExtract
import docspell.joex.scheduler._
import docspell.store.records.RAttachment
import docspell.store.records._
import docspell.store.syntax.MimeTypes._
import bitpeace.{Mimetype, RangeDef}
/** Goes through all attachments that must be already converted into a
* pdf. If it is a pdf, the number of pages are retrieved and stored
* in the attachment metadata.
*/
object AttachmentPageCount {
def apply[F[_]: Sync: ContextShift]()(
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
for {
_ <- ctx.logger.info(
s"Retrieving page count for ${item.attachments.size} files…"
)
_ <- item.attachments
.traverse(createPageCount(ctx))
.attempt
.flatMap {
case Right(_) => ().pure[F]
case Left(ex) =>
ctx.logger.error(ex)(
s"Retrieving page counts failed, continuing without it."
)
}
} yield item
}
def createPageCount[F[_]: Sync](
ctx: Context[F, _]
)(ra: RAttachment): F[Option[PdfMetaData]] =
findMime[F](ctx)(ra).flatMap {
case MimeType.PdfMatch(_) =>
PdfboxExtract.getMetaData(loadFile(ctx)(ra)).flatMap {
case Right(md) =>
ctx.logger.debug(s"Found number of pages: ${md.pageCount}") *>
updatePageCount(ctx, md, ra).map(_.some)
case Left(ex) =>
ctx.logger.warn(s"Error obtaining pages count: ${ex.getMessage}") *>
(None: Option[PdfMetaData]).pure[F]
}
case mt =>
ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot get page count.") *>
(None: Option[PdfMetaData]).pure[F]
}
private def updatePageCount[F[_]: Sync](
ctx: Context[F, _],
md: PdfMetaData,
ra: RAttachment
): F[PdfMetaData] =
for {
_ <- ctx.logger.debug(
s"Update attachment ${ra.id.id} with page count ${md.pageCount.some}"
)
n <- ctx.store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some))
m <-
if (n == 0)
ctx.logger.warn(
s"No attachmentmeta record exists for ${ra.id.id}. Creating new."
) *> ctx.store.transact(
RAttachmentMeta.insert(
RAttachmentMeta(ra.id, None, Nil, MetaProposalList.empty, md.pageCount.some)
)
)
else 0.pure[F]
_ <- ctx.logger.debug(s"Stored page count (${n + m}).")
} yield md
def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] =
OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId)))
.map(_.mimetype)
.getOrElse(Mimetype.`application/octet-stream`)
.map(_.toLocal)
def loadFile[F[_]](ctx: Context[F, _])(ra: RAttachment): Stream[F, Byte] =
ctx.store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.through(ctx.store.bitpeace.fetchData2(RangeDef.all))
}

View File

@ -55,6 +55,7 @@ object ProcessItem {
.flatMap(Task.setProgress(progress._1))
.flatMap(TextExtraction(cfg.extraction, fts))
.flatMap(AttachmentPreview(cfg.convert, cfg.extraction.preview))
.flatMap(AttachmentPageCount())
.flatMap(Task.setProgress(progress._2))
.flatMap(analysisOnly[F](cfg, analyser, regexNer))
.flatMap(Task.setProgress(progress._3))

View File

@ -1291,10 +1291,11 @@ paths:
summary: Search for items.
description: |
Search for items given a search form. The results are grouped
by month and are sorted by item date (newest first). Tags are
*not* resolved. The results will always contain an empty list
for item tags. Use `/searchWithTags` to also retrieve all tags
of an item.
by month and are sorted by item date (newest first). Tags and
attachments are *not* resolved. The results will always
contain an empty list for item tags and attachments. Use
`/searchWithTags` to also retrieve all tags and a list of
attachments of an item.
The `fulltext` field can be used to restrict the results by
using full-text search in the documents contents.
@ -1318,9 +1319,10 @@ paths:
summary: Search for items.
description: |
Search for items given a search form. The results are grouped
by month by default. For each item, its tags are also
returned. This uses more queries and is therefore slower, but
returns all tags to an item.
by month by default. For each item, its tags and attachments
are also returned. This uses more queries and is therefore
slower, but returns all tags to an item as well as their
attachments with some minor details.
The `fulltext` field can be used to restrict the results by
using full-text search in the documents contents.
@ -4703,6 +4705,10 @@ components:
fileCount:
type: integer
format: int32
attachments:
type: array
items:
$ref: "#/components/schemas/AttachmentLight"
tags:
type: array
items:
@ -4720,6 +4726,24 @@ components:
type: array
items:
$ref: "#/components/schemas/HighlightEntry"
AttachmentLight:
description: |
Some little details about an attachment.
required:
- id
- position
properties:
id:
type: string
format: ident
position:
type: integer
format: int32
name:
type: string
pageCount:
type: integer
format: int32
HighlightEntry:
description: |
Highlighting information for a single field (maybe attachment

View File

@ -15,6 +15,7 @@ import docspell.common.syntax.all._
import docspell.ftsclient.FtsResult
import docspell.restapi.model._
import docspell.restserver.conv.Conversions._
import docspell.store.queries.QItem
import docspell.store.records._
import docspell.store.{AddResult, UpdateResult}
@ -204,6 +205,7 @@ trait Conversions {
i.folder.map(mkIdName),
i.fileCount,
Nil,
Nil,
i.notes,
Nil
)
@ -215,7 +217,11 @@ trait Conversions {
}
def mkItemLightWithTags(i: OItemSearch.ListItemWithTags): ItemLight =
mkItemLight(i.item).copy(tags = i.tags.map(mkTag))
mkItemLight(i.item)
.copy(tags = i.tags.map(mkTag), attachments = i.attachments.map(mkAttachmentLight))
private def mkAttachmentLight(qa: QItem.AttachmentLight): AttachmentLight =
AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount)
def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = {
val il = mkItemLightWithTags(i.item)

View File

@ -0,0 +1,2 @@
ALTER TABLE "attachmentmeta"
ADD COLUMN "page_count" smallint;

View File

@ -0,0 +1,2 @@
ALTER TABLE `attachmentmeta`
ADD COLUMN (`page_count` SMALLINT);

View File

@ -0,0 +1,2 @@
ALTER TABLE "attachmentmeta"
ADD COLUMN "page_count" smallint;

View File

@ -443,7 +443,17 @@ object QItem {
from.query[ListItem].stream
}
case class ListItemWithTags(item: ListItem, tags: List[RTag])
case class AttachmentLight(
id: Ident,
position: Int,
name: Option[String],
pageCount: Option[Int]
)
case class ListItemWithTags(
item: ListItem,
tags: List[RTag],
attachments: List[AttachmentLight]
)
/** Same as `findItems` but resolves the tags for each item. Note that
* this is implemented by running an additional query per item.
@ -476,8 +486,29 @@ object QItem {
item <- search
tagItems <- Stream.eval(RTagItem.findByItem(item.id))
tags <- Stream.eval(tagItems.traverse(ti => findTag(resolvedTags, ti)))
attachs <- Stream.eval(findAttachmentLight(item.id))
ftags = tags.flatten.filter(t => t.collective == collective)
} yield ListItemWithTags(item, ftags.toList.sortBy(_.name))
} yield ListItemWithTags(
item,
ftags.toList.sortBy(_.name),
attachs.sortBy(_.position)
)
}
private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = {
val aId = RAttachment.Columns.id.prefix("a")
val aItem = RAttachment.Columns.itemId.prefix("a")
val aPos = RAttachment.Columns.position.prefix("a")
val aName = RAttachment.Columns.name.prefix("a")
val mId = RAttachmentMeta.Columns.id.prefix("m")
val mPages = RAttachmentMeta.Columns.pages.prefix("m")
val cols = Seq(aId, aPos, aName, mPages)
val join = RAttachment.table ++
fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId)
val cond = aItem.is(item)
selectSimple(cols, join, cond).query[AttachmentLight].to[List]
}
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =

View File

@ -255,15 +255,32 @@ object RAttachment {
}
}
def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = {
val aId = Columns.id.prefix("a")
val aCreated = Columns.created.prefix("a")
val mId = RAttachmentMeta.Columns.id.prefix("m")
val mPages = RAttachmentMeta.Columns.pages.prefix("m")
val cols = all.map(_.prefix("a"))
val join = table ++ fr"a LEFT OUTER JOIN" ++
RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId)
val cond = mPages.isNull
(selectSimple(cols, join, cond) ++ orderBy(aCreated.desc))
.query[RAttachment]
.streamWithChunkSize(chunkSize)
}
def findWithoutPreview(
coll: Option[Ident],
chunkSize: Int
): Stream[ConnectionIO, RAttachment] = {
val aId = Columns.id.prefix("a")
val aItem = Columns.itemId.prefix("a")
val pId = RAttachmentPreview.Columns.id.prefix("p")
val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i")
val aId = Columns.id.prefix("a")
val aItem = Columns.itemId.prefix("a")
val aCreated = Columns.created.prefix("a")
val pId = RAttachmentPreview.Columns.id.prefix("p")
val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i")
val cols = all.map(_.prefix("a"))
val baseJoin =
@ -277,11 +294,11 @@ object RAttachment {
case Some(cid) =>
val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem)
val cond = and(baseCond ++ Seq(iColl.is(cid)))
selectSimple(cols, join, cond)
(selectSimple(cols, join, cond) ++ orderBy(aCreated.desc))
.query[RAttachment]
.streamWithChunkSize(chunkSize)
case None =>
selectSimple(cols, baseJoin, and(baseCond))
(selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc))
.query[RAttachment]
.streamWithChunkSize(chunkSize)
}

View File

@ -13,17 +13,21 @@ case class RAttachmentMeta(
id: Ident, //same as RAttachment.id
content: Option[String],
nerlabels: List[NerLabel],
proposals: MetaProposalList
proposals: MetaProposalList,
pages: Option[Int]
) {
def setContentIfEmpty(txt: Option[String]): RAttachmentMeta =
if (content.forall(_.trim.isEmpty)) copy(content = txt)
else this
def withPageCount(count: Option[Int]): RAttachmentMeta =
copy(pages = count)
}
object RAttachmentMeta {
def empty(attachId: Ident) =
RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty)
RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty, None)
val table = fr"attachmentmeta"
@ -32,7 +36,8 @@ object RAttachmentMeta {
val content = Column("content")
val nerlabels = Column("nerlabels")
val proposals = Column("itemproposals")
val all = List(id, content, nerlabels, proposals)
val pages = Column("page_count")
val all = List(id, content, nerlabels, proposals, pages)
}
import Columns._
@ -40,7 +45,7 @@ object RAttachmentMeta {
insertRow(
table,
all,
fr"${v.id},${v.content},${v.nerlabels},${v.proposals}"
fr"${v.id},${v.content},${v.nerlabels},${v.proposals},${v.pages}"
).update.run
def exists(attachId: Ident): ConnectionIO[Boolean] =
@ -49,6 +54,12 @@ object RAttachmentMeta {
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentMeta]] =
selectSimple(all, table, id.is(attachId)).query[RAttachmentMeta].option
def findPageCountById(attachId: Ident): ConnectionIO[Option[Int]] =
selectSimple(Seq(pages), table, id.is(attachId))
.query[Option[Int]]
.option
.map(_.flatten)
def upsert(v: RAttachmentMeta): ConnectionIO[Int] =
for {
n0 <- update(v)
@ -84,6 +95,9 @@ object RAttachmentMeta {
)
).update.run
def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] =
updateRow(table, id.is(mid), pages.setTo(pageCount)).update.run
def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id.is(attachId)).update.run
}

View File

@ -6,6 +6,7 @@ module Api exposing
, addMember
, addTag
, addTagsMultiple
, attachmentPreviewURL
, cancelJob
, changeFolderName
, changePassword
@ -58,9 +59,9 @@ module Api exposing
, getTagCloud
, getTags
, getUsers
, itemBasePreviewURL
, itemDetail
, itemIndexSearch
, itemPreviewURL
, itemSearch
, login
, loginSession
@ -144,6 +145,7 @@ import Api.Model.InviteResult exposing (InviteResult)
import Api.Model.ItemDetail exposing (ItemDetail)
import Api.Model.ItemFtsSearch exposing (ItemFtsSearch)
import Api.Model.ItemInsights exposing (ItemInsights)
import Api.Model.ItemLight exposing (ItemLight)
import Api.Model.ItemLightList exposing (ItemLightList)
import Api.Model.ItemProposals exposing (ItemProposals)
import Api.Model.ItemSearch exposing (ItemSearch)
@ -1503,8 +1505,13 @@ deleteAllItems flags ids receive =
--- Item
itemPreviewURL : String -> String
itemPreviewURL itemId =
attachmentPreviewURL : String -> String
attachmentPreviewURL id =
"/api/v1/sec/attachment/" ++ id ++ "/preview?withFallback=true"
itemBasePreviewURL : String -> String
itemBasePreviewURL itemId =
"/api/v1/sec/item/" ++ itemId ++ "/preview?withFallback=true"

View File

@ -0,0 +1,445 @@
module Comp.ItemCard exposing (..)
import Api
import Api.Model.AttachmentLight exposing (AttachmentLight)
import Api.Model.HighlightEntry exposing (HighlightEntry)
import Api.Model.ItemLight exposing (ItemLight)
import Data.Direction
import Data.Fields
import Data.Icons as Icons
import Data.ItemSelection exposing (ItemSelection)
import Data.UiSettings exposing (UiSettings)
import Html exposing (..)
import Html.Attributes exposing (..)
import Html.Events exposing (onClick)
import Markdown
import Page exposing (Page(..))
import Set exposing (Set)
import Util.Html
import Util.ItemDragDrop as DD
import Util.List
import Util.Maybe
import Util.String
import Util.Time
type alias Model =
{ previewAttach : Maybe AttachmentLight
}
type Msg
= CyclePreview ItemLight
| ToggleSelectItem (Set String) String
| ItemDDMsg DD.Msg
type alias ViewConfig =
{ selection : ItemSelection
, extraClasses : String
}
type alias UpdateResult =
{ model : Model
, dragModel : DD.Model
, selection : ItemSelection
}
init : Model
init =
{ previewAttach = Nothing
}
currentAttachment : Model -> ItemLight -> Maybe AttachmentLight
currentAttachment model item =
Util.Maybe.or
[ model.previewAttach
, List.head item.attachments
]
currentPosition : Model -> ItemLight -> Int
currentPosition model item =
let
filter cur el =
cur.id == el.id
in
case model.previewAttach of
Just a ->
case Util.List.findIndexed (filter a) item.attachments of
Just ( _, n ) ->
n + 1
Nothing ->
1
Nothing ->
1
update : DD.Model -> Msg -> Model -> UpdateResult
update ddm msg model =
case msg of
ItemDDMsg lm ->
let
ddd =
DD.update lm ddm
in
UpdateResult model ddd.model Data.ItemSelection.Inactive
ToggleSelectItem ids id ->
let
newSet =
if Set.member id ids then
Set.remove id ids
else
Set.insert id ids
in
UpdateResult model ddm (Data.ItemSelection.Active newSet)
CyclePreview item ->
let
mainAttach =
currentAttachment model item
next =
Util.List.findNext (\e -> Just e.id == Maybe.map .id mainAttach) item.attachments
in
UpdateResult { model | previewAttach = next }
ddm
Data.ItemSelection.Inactive
view : ViewConfig -> UiSettings -> Model -> ItemLight -> Html Msg
view cfg settings model item =
let
dirIcon =
i [ class (Data.Direction.iconFromMaybe item.direction) ] []
corr =
List.filterMap identity [ item.corrOrg, item.corrPerson ]
|> List.map .name
|> List.intersperse ", "
|> String.concat
conc =
List.filterMap identity [ item.concPerson, item.concEquip ]
|> List.map .name
|> List.intersperse ", "
|> String.concat
folder =
Maybe.map .name item.folder
|> Maybe.withDefault ""
dueDate =
Maybe.map Util.Time.formatDateShort item.dueDate
|> Maybe.withDefault ""
isConfirmed =
item.state /= "created"
cardColor =
if isSelected cfg item.id then
"purple"
else if not isConfirmed then
"blue"
else
""
fieldHidden f =
Data.UiSettings.fieldHidden settings f
cardAction =
case cfg.selection of
Data.ItemSelection.Inactive ->
Page.href (ItemDetailPage item.id)
Data.ItemSelection.Active ids ->
onClick (ToggleSelectItem ids item.id)
mainAttach =
currentAttachment model item
previewUrl =
Maybe.map .id mainAttach
|> Maybe.map Api.attachmentPreviewURL
|> Maybe.withDefault (Api.itemBasePreviewURL item.id)
pageCount =
Maybe.andThen .pageCount mainAttach
|> Maybe.withDefault 0
pageCountLabel =
div
[ classList
[ ( "card-attachment-nav top", True )
, ( "invisible", pageCount == 0 || (item.fileCount == 1 && pageCount == 1) )
]
]
[ if item.fileCount == 1 then
div
[ class "ui secondary basic mini label"
, title "Number of pages"
]
[ text "p."
, text (String.fromInt pageCount)
]
else
div [ class "ui left labeled mini button" ]
[ div [ class "ui basic right pointing mini label" ]
[ currentPosition model item
|> String.fromInt
|> text
, text "/"
, text (String.fromInt item.fileCount)
, text " p."
, text (String.fromInt pageCount)
]
, a
[ class "ui mini icon secondary button"
, href "#"
, onClick (CyclePreview item)
]
[ i [ class "arrow right icon" ] []
]
]
]
in
div
([ classList
[ ( "ui fluid card", True )
, ( cardColor, True )
, ( cfg.extraClasses, True )
]
, id item.id
]
++ DD.draggable ItemDDMsg item.id
)
[ if fieldHidden Data.Fields.PreviewImage then
span [ class "invisible" ] []
else
div
[ class "image ds-card-image"
, Data.UiSettings.cardPreviewSize settings
]
[ img
[ class "preview-image"
, src previewUrl
, Data.UiSettings.cardPreviewSize settings
]
[]
, pageCountLabel
]
, a
[ class "link content"
, href "#"
, cardAction
]
[ case cfg.selection of
Data.ItemSelection.Active ids ->
div [ class "header" ]
[ Util.Html.checkbox (Set.member item.id ids)
, dirIcon
, Util.String.underscoreToSpace item.name
|> text
]
Data.ItemSelection.Inactive ->
if fieldHidden Data.Fields.Direction then
div [ class "header" ]
[ Util.String.underscoreToSpace item.name |> text
]
else
div
[ class "header"
, Data.Direction.labelFromMaybe item.direction
|> title
]
[ dirIcon
, Util.String.underscoreToSpace item.name
|> text
]
, div
[ classList
[ ( "ui right corner label", True )
, ( cardColor, True )
, ( "invisible", isConfirmed )
]
, title "New"
]
[ i [ class "exclamation icon" ] []
]
, div
[ classList
[ ( "meta", True )
, ( "invisible hidden", fieldHidden Data.Fields.Date )
]
]
[ Util.Time.formatDate item.date |> text
]
, div [ class "meta description" ]
[ div
[ classList
[ ( "ui right floated tiny labels", True )
, ( "invisible hidden", item.tags == [] || fieldHidden Data.Fields.Tag )
]
]
(List.map
(\tag ->
div
[ classList
[ ( "ui basic label", True )
, ( Data.UiSettings.tagColorString tag settings, True )
]
]
[ text tag.name ]
)
item.tags
)
]
]
, div
[ classList
[ ( "content", True )
, ( "invisible hidden"
, settings.itemSearchNoteLength
<= 0
|| Util.String.isNothingOrBlank item.notes
)
]
]
[ span [ class "small-info" ]
[ Maybe.withDefault "" item.notes
|> Util.String.ellipsis settings.itemSearchNoteLength
|> text
]
]
, div [ class "content" ]
[ div [ class "ui horizontal list" ]
[ div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, fieldHidden Data.Fields.CorrOrg
&& fieldHidden Data.Fields.CorrPerson
)
]
, title "Correspondent"
]
[ Icons.correspondentIcon ""
, text " "
, Util.String.withDefault "-" corr |> text
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, fieldHidden Data.Fields.ConcPerson
&& fieldHidden Data.Fields.ConcEquip
)
]
, title "Concerning"
]
[ Icons.concernedIcon
, text " "
, Util.String.withDefault "-" conc |> text
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden", fieldHidden Data.Fields.Folder )
]
, title "Folder"
]
[ Icons.folderIcon ""
, text " "
, Util.String.withDefault "-" folder |> text
]
]
, div [ class "right floated meta" ]
[ div [ class "ui horizontal list" ]
[ div
[ class "item"
, title "Source"
]
[ text item.source
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, item.dueDate
== Nothing
|| fieldHidden Data.Fields.DueDate
)
]
, title ("Due on " ++ dueDate)
]
[ div
[ class "ui basic grey label"
]
[ Icons.dueDateIcon ""
, text (" " ++ dueDate)
]
]
]
]
]
, div
[ classList
[ ( "content search-highlight", True )
, ( "invisible hidden", item.highlighting == [] )
]
]
[ div [ class "ui list" ]
(List.map renderHighlightEntry item.highlighting)
]
]
renderHighlightEntry : HighlightEntry -> Html Msg
renderHighlightEntry entry =
let
stripWhitespace str =
String.trim str
|> String.replace "```" ""
|> String.replace "\t" " "
|> String.replace "\n\n" "\n"
|> String.lines
|> List.map String.trim
|> String.join "\n"
in
div [ class "item" ]
[ div [ class "content" ]
(div [ class "header" ]
[ i [ class "caret right icon" ] []
, text (entry.name ++ ":")
]
:: List.map
(\str ->
Markdown.toHtml [ class "description" ] <|
(stripWhitespace str ++ "")
)
entry.lines
)
]
isSelected : ViewConfig -> String -> Bool
isSelected cfg id =
case cfg.selection of
Data.ItemSelection.Active ids ->
Set.member id ids
Data.ItemSelection.Inactive ->
False

View File

@ -10,46 +10,38 @@ module Comp.ItemCardList exposing
, view
)
import Api
import Api.Model.HighlightEntry exposing (HighlightEntry)
import Api.Model.ItemLight exposing (ItemLight)
import Api.Model.ItemLightGroup exposing (ItemLightGroup)
import Api.Model.ItemLightList exposing (ItemLightList)
import Data.Direction
import Data.Fields
import Comp.ItemCard
import Data.Flags exposing (Flags)
import Data.Icons as Icons
import Data.ItemSelection exposing (ItemSelection)
import Data.Items
import Data.UiSettings exposing (UiSettings)
import Dict exposing (Dict)
import Html exposing (..)
import Html.Attributes exposing (..)
import Html.Events exposing (onClick)
import Markdown
import Page exposing (Page(..))
import Set exposing (Set)
import Util.Html
import Util.ItemDragDrop as DD
import Util.List
import Util.String
import Util.Time
type alias Model =
{ results : ItemLightList
, itemCards : Dict String Comp.ItemCard.Model
}
type Msg
= SetResults ItemLightList
| AddResults ItemLightList
| ItemDDMsg DD.Msg
| ToggleSelectItem (Set String) String
| ItemCardMsg ItemLight Comp.ItemCard.Msg
init : Model
init =
{ results = Api.Model.ItemLightList.empty
, itemCards = Dict.empty
}
@ -112,23 +104,22 @@ updateDrag dm _ msg model =
in
UpdateResult newModel Cmd.none dm Data.ItemSelection.Inactive
ItemDDMsg lm ->
ItemCardMsg item lm ->
let
ddd =
DD.update lm dm
in
UpdateResult model Cmd.none ddd.model Data.ItemSelection.Inactive
cardModel =
Dict.get item.id model.itemCards
|> Maybe.withDefault Comp.ItemCard.init
ToggleSelectItem ids id ->
let
newSet =
if Set.member id ids then
Set.remove id ids
result =
Comp.ItemCard.update dm lm cardModel
else
Set.insert id ids
cards =
Dict.insert item.id result.model model.itemCards
in
UpdateResult model Cmd.none dm (Data.ItemSelection.Active newSet)
UpdateResult { model | itemCards = cards }
Cmd.none
result.dragModel
result.selection
@ -141,295 +132,42 @@ type alias ViewConfig =
}
isSelected : ViewConfig -> String -> Bool
isSelected cfg id =
case cfg.selection of
Data.ItemSelection.Active ids ->
Set.member id ids
Data.ItemSelection.Inactive ->
False
view : ViewConfig -> UiSettings -> Model -> Html Msg
view cfg settings model =
div [ class "ui container" ]
(List.map (viewGroup cfg settings) model.results.groups)
(List.map (viewGroup model cfg settings) model.results.groups)
viewGroup : ViewConfig -> UiSettings -> ItemLightGroup -> Html Msg
viewGroup cfg settings group =
viewGroup : Model -> ViewConfig -> UiSettings -> ItemLightGroup -> Html Msg
viewGroup model cfg settings group =
div [ class "item-group" ]
[ div [ class "ui horizontal divider header item-list" ]
[ i [ class "calendar alternate outline icon" ] []
, text group.name
]
, div [ class "ui stackable three cards" ]
(List.map (viewItem cfg settings) group.items)
(List.map (viewItem model cfg settings) group.items)
]
viewItem : ViewConfig -> UiSettings -> ItemLight -> Html Msg
viewItem cfg settings item =
viewItem : Model -> ViewConfig -> UiSettings -> ItemLight -> Html Msg
viewItem model cfg settings item =
let
dirIcon =
i [ class (Data.Direction.iconFromMaybe item.direction) ] []
corr =
List.filterMap identity [ item.corrOrg, item.corrPerson ]
|> List.map .name
|> List.intersperse ", "
|> String.concat
conc =
List.filterMap identity [ item.concPerson, item.concEquip ]
|> List.map .name
|> List.intersperse ", "
|> String.concat
folder =
Maybe.map .name item.folder
|> Maybe.withDefault ""
dueDate =
Maybe.map Util.Time.formatDateShort item.dueDate
|> Maybe.withDefault ""
isConfirmed =
item.state /= "created"
cardColor =
if isSelected cfg item.id then
"purple"
else if not isConfirmed then
"blue"
currentClass =
if cfg.current == Just item.id then
"current"
else
""
fieldHidden f =
Data.UiSettings.fieldHidden settings f
vvcfg =
Comp.ItemCard.ViewConfig cfg.selection currentClass
cardAction =
case cfg.selection of
Data.ItemSelection.Inactive ->
Page.href (ItemDetailPage item.id)
cardModel =
Dict.get item.id model.itemCards
|> Maybe.withDefault Comp.ItemCard.init
Data.ItemSelection.Active ids ->
onClick (ToggleSelectItem ids item.id)
cardHtml =
Comp.ItemCard.view vvcfg settings cardModel item
in
a
([ classList
[ ( "ui fluid card", True )
, ( cardColor, True )
, ( "current", cfg.current == Just item.id )
]
, id item.id
, href "#"
, cardAction
]
++ DD.draggable ItemDDMsg item.id
)
[ if fieldHidden Data.Fields.PreviewImage then
span [ class "invisible" ] []
else
div [ class "image" ]
[ img
[ class "preview-image"
, src (Api.itemPreviewURL item.id)
, Data.UiSettings.cardPreviewSize settings
]
[]
]
, div [ class "content" ]
[ case cfg.selection of
Data.ItemSelection.Active ids ->
div [ class "header" ]
[ Util.Html.checkbox (Set.member item.id ids)
, dirIcon
, Util.String.underscoreToSpace item.name
|> text
]
Data.ItemSelection.Inactive ->
if fieldHidden Data.Fields.Direction then
div [ class "header" ]
[ Util.String.underscoreToSpace item.name |> text
]
else
div
[ class "header"
, Data.Direction.labelFromMaybe item.direction
|> title
]
[ dirIcon
, Util.String.underscoreToSpace item.name
|> text
]
, div
[ classList
[ ( "ui right corner label", True )
, ( cardColor, True )
, ( "invisible", isConfirmed )
]
, title "New"
]
[ i [ class "exclamation icon" ] []
]
, div
[ classList
[ ( "meta", True )
, ( "invisible hidden", fieldHidden Data.Fields.Date )
]
]
[ Util.Time.formatDate item.date |> text
]
, div [ class "meta description" ]
[ div
[ classList
[ ( "ui right floated tiny labels", True )
, ( "invisible hidden", item.tags == [] || fieldHidden Data.Fields.Tag )
]
]
(List.map
(\tag ->
div
[ classList
[ ( "ui basic label", True )
, ( Data.UiSettings.tagColorString tag settings, True )
]
]
[ text tag.name ]
)
item.tags
)
]
]
, div
[ classList
[ ( "content", True )
, ( "invisible hidden"
, settings.itemSearchNoteLength
<= 0
|| Util.String.isNothingOrBlank item.notes
)
]
]
[ span [ class "small-info" ]
[ Maybe.withDefault "" item.notes
|> Util.String.ellipsis settings.itemSearchNoteLength
|> text
]
]
, div [ class "content" ]
[ div [ class "ui horizontal list" ]
[ div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, fieldHidden Data.Fields.CorrOrg
&& fieldHidden Data.Fields.CorrPerson
)
]
, title "Correspondent"
]
[ Icons.correspondentIcon ""
, text " "
, Util.String.withDefault "-" corr |> text
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, fieldHidden Data.Fields.ConcPerson
&& fieldHidden Data.Fields.ConcEquip
)
]
, title "Concerning"
]
[ Icons.concernedIcon
, text " "
, Util.String.withDefault "-" conc |> text
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden", fieldHidden Data.Fields.Folder )
]
, title "Folder"
]
[ Icons.folderIcon ""
, text " "
, Util.String.withDefault "-" folder |> text
]
]
, div [ class "right floated meta" ]
[ div [ class "ui horizontal list" ]
[ div
[ class "item"
, title "Source"
]
[ text item.source
]
, div
[ classList
[ ( "item", True )
, ( "invisible hidden"
, item.dueDate
== Nothing
|| fieldHidden Data.Fields.DueDate
)
]
, title ("Due on " ++ dueDate)
]
[ div
[ class "ui basic grey label"
]
[ Icons.dueDateIcon ""
, text (" " ++ dueDate)
]
]
]
]
]
, div
[ classList
[ ( "content search-highlight", True )
, ( "invisible hidden", item.highlighting == [] )
]
]
[ div [ class "ui list" ]
(List.map renderHighlightEntry item.highlighting)
]
]
renderHighlightEntry : HighlightEntry -> Html Msg
renderHighlightEntry entry =
let
stripWhitespace str =
String.trim str
|> String.replace "```" ""
|> String.replace "\t" " "
|> String.replace "\n\n" "\n"
|> String.lines
|> List.map String.trim
|> String.join "\n"
in
div [ class "item" ]
[ div [ class "content" ]
(div [ class "header" ]
[ i [ class "caret right icon" ] []
, text (entry.name ++ ":")
]
:: List.map
(\str ->
Markdown.toHtml [ class "description" ] <|
(stripWhitespace str ++ "")
)
entry.lines
)
]
Html.map (ItemCardMsg item) cardHtml

View File

@ -226,15 +226,8 @@ fieldHidden settings field =
cardPreviewSize : UiSettings -> Attribute msg
cardPreviewSize settings =
case settings.cardPreviewSize of
Data.BasicSize.Small ->
HA.style "max-width" "80px"
Data.BasicSize.Medium ->
HA.style "max-width" "160px"
Data.BasicSize.Large ->
HA.style "max-width" "none"
Data.BasicSize.asString settings.cardPreviewSize
|> HA.class

View File

@ -307,7 +307,7 @@ viewSearchBar flags model =
]
[ i [ class "filter icon" ] []
]
, div [ class "item" ]
, div [ class "right fitted item" ]
[ div [ class "ui left icon right action input" ]
[ i
[ classList

View File

@ -93,10 +93,45 @@
padding: 0.8em;
}
.default-layout .ui.card .link.content:hover {
box-shadow: 0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);
}
.default-layout .image .card-attachment-nav {
position: absolute;
right: 2px;
z-index: 10;
}
.default-layout .image .card-attachment-nav.bottom {
bottom: 2px;
}
.default-layout .image .card-attachment-nav.top {
top: 2px;
}
.default-layout .image.ds-card-image {
overflow: auto;
}
.default-layout .image.ds-card-image.small {
max-height: 120px;
}
.default-layout .image.ds-card-image.medium {
max-height: 240px;
}
.default-layout .image.ds-card-image.large {
max-height: 600px;
}
.default-layout img.preview-image {
margin-left: auto;
margin-right: auto;
}
.default-layout img.preview-image.small {
max-width: 80px;
}
.default-layout img.preview-image.medium {
max-width: 160px;
}
.default-layout img.preview-image.large {
max-width: none;
}
.default-layout .menu .item.active a.right-tab-icon-link {
position: relative;
@ -119,7 +154,6 @@
background: rgba(220, 255, 71, 0.6);
}
.default-layout .ui.cards .ui.card.current {
/* semantic-ui purple */
box-shadow: 0 0 6px rgba(0,0,0,0.55);
}