TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-21 18:08:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix finding attachments for retries

Browse Source 
The attachments to process again must be searched in sources and
archives, too.
This commit is contained in:
Eike Kettner
2020-10-02 00:19:25 +02:00
parent 5e21552358
commit b6f23b038a
3 changed files with 55 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
View File

@ -1,5 +1,6 @@
package docspell.joex.process
import cats.data.NonEmptyList
import cats.data.OptionT
import cats.effect.Sync
import cats.implicits._
@ -125,21 +126,31 @@ object CreateItem {
for {
cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq))
_ <-
if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
if (cand.nonEmpty)
ctx.logger.warn(s"Found ${cand.size} existing item with these files.")
else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <-
if (ht.sum > 0)
ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
else ().pure[F]
rms <- OptionT(
//load attachments but only those mentioned in the task's arguments
cand.headOption.traverse(ri =>
ctx.store
.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
.map(_.filter(r => fileMetaIds.contains(r.fileId)))
rms <- OptionT
.fromOption[F](NonEmptyList.fromList(fileMetaIds.toList))
.flatMap(fids =>
OptionT(
//load attachments but only those mentioned in the task's arguments
cand.headOption.traverse(ri =>
ctx.store
.transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids))
.flatTap(ats =>
ctx.logger.debug(
s"Found ${ats.size} attachments. Use only those from task args: ${fileMetaIds}"
)
)
)
)
)
).getOrElse(Vector.empty)
.getOrElse(Vector.empty)
orig <- rms.traverse(a =>
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
)

7
modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
View File

@ -20,7 +20,9 @@ object TextExtraction {
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
for {
_ <- ctx.logger.info("Starting text extraction")
_ <- ctx.logger.info(
s"Starting text extraction for ${item.attachments.size} files"
)
start <- Duration.stopTime[F]
txt <- item.attachments.traverse(
extractTextIfEmpty(
@ -31,9 +33,10 @@ object TextExtraction {
item
)
)
_ <- ctx.logger.debug("Storing extracted texts")
_ <- ctx.logger.debug("Storing extracted texts …")
_ <-
txt.toList.traverse(res => ctx.store.transact(RAttachmentMeta.upsert(res.am)))
_ <- ctx.logger.debug(s"Extracted text stored.")
idxItem = TextData.item(
item.item.id,
ctx.args.meta.collective,