Allow to specify an item id to amend files to existing items

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -77,7 +77,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             ProcessItemArgs.taskName,
-            ItemHandler[F](cfg),
+            ItemHandler.newItem[F](cfg),
             ItemHandler.onCancel[F]
           )
         )

modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala

@@ -32,44 +32,75 @@ object CreateItem {
 
       def fileMetas(itemId: Ident, now: Timestamp) =
         Stream
-          .emits(ctx.args.files)
-          .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
-          .collect({ case (f, Some(fm)) if isValidFile(fm) => f })
-          .zipWithIndex
-          .evalMap({
-            case (f, index) =>
-              Ident
-                .randomId[F]
-                .map(id =>
-                  RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)
-                )
-          })
+          .eval(ctx.store.transact(RAttachment.countOnItem(itemId)))
+          .flatMap { offset =>
+            Stream
+              .emits(ctx.args.files)
+              .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
+              .collect({ case (f, Some(fm)) if isValidFile(fm) => f })
+              .zipWithIndex
+              .evalMap({
+                case (f, index) =>
+                  Ident
+                    .randomId[F]
+                    .map(id =>
+                      RAttachment(
+                        id,
+                        itemId,
+                        f.fileMetaId,
+                        index.toInt + offset,
+                        now,
+                        f.name
+                      )
+                    )
+              })
+          }
           .compile
           .toVector
 
-      val item = RItem.newItem[F](
-        ctx.args.meta.collective,
-        ctx.args.makeSubject,
-        ctx.args.meta.sourceAbbrev,
-        ctx.args.meta.direction.getOrElse(Direction.Incoming),
-        ItemState.Premature
-      )
+      val loadItemOrInsertNew =
+        ctx.args.meta.itemId match {
+          case Some(id) =>
+            (for {
+              _ <- OptionT.liftF(
+                ctx.logger.info(
+                  s"Loading item with id ${id.id} to ammend"
+                )
+              )
+              item <- OptionT(
+                ctx.store
+                  .transact(RItem.findByIdAndCollective(id, ctx.args.meta.collective))
+              )
+            } yield (1, item))
+              .getOrElseF(Sync[F].raiseError(new Exception(s"Item not found.")))
+          case None =>
+            for {
+              _ <- ctx.logger.info(
+                s"Creating new item with ${ctx.args.files.size} attachment(s)"
+              )
+              item <- RItem.newItem[F](
+                ctx.args.meta.collective,
+                ctx.args.makeSubject,
+                ctx.args.meta.sourceAbbrev,
+                ctx.args.meta.direction.getOrElse(Direction.Incoming),
+                ItemState.Premature
+              )
+              n <- ctx.store.transact(RItem.insert(item))
+            } yield (n, item)
+        }
 
       for {
-        _ <- ctx.logger.info(
-          s"Creating new item with ${ctx.args.files.size} attachment(s)"
-        )
         time <- Duration.stopTime[F]
-        it   <- item
-        n    <- ctx.store.transact(RItem.insert(it))
-        _    <- if (n != 1) storeItemError[F](ctx) else ().pure[F]
-        fm   <- fileMetas(it.id, it.created)
+        it   <- loadItemOrInsertNew
+        _    <- if (it._1 != 1) storeItemError[F](ctx) else ().pure[F]
+        now  <- Timestamp.current[F]
+        fm   <- fileMetas(it._2.id, now)
         k    <- fm.traverse(insertAttachment(ctx))
         _    <- logDifferences(ctx, fm, k.sum)
         dur  <- time
         _    <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}")
       } yield ItemData(
-        it,
+        it._2,
         fm,
         Vector.empty,
         Vector.empty,
@@ -86,7 +117,7 @@ object CreateItem {
     } yield n)
   }
 
-  def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
+  private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
     Task { ctx =>
       for {
         cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))

modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala

@@ -50,8 +50,10 @@ object ExtractArchive {
         findMime(ctx)(ra).flatMap(m => extractSafe(ctx, archive)(ra, m))
 
       for {
-        ras <- item.attachments.traverse(extract)
-        nra = ras.flatMap(_.files).zipWithIndex.map(t => t._1.copy(position = t._2))
+        ras     <- item.attachments.traverse(extract)
+        lastPos <- ctx.store.transact(RAttachment.countOnItem(item.item.id))
+        nra =
+          ras.flatMap(_.files).zipWithIndex.map(t => t._1.copy(position = lastPos + t._2))
         _ <- nra.traverse(storeAttachment(ctx))
         naa = ras.flatMap(_.archives)
         _ <- naa.traverse(storeArchive(ctx))

modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala

@@ -2,19 +2,20 @@ package docspell.joex.process
 
 import cats.implicits._
 import cats.effect._
+import fs2.Stream
 import docspell.common.{ItemState, ProcessItemArgs}
 import docspell.joex.Config
-import docspell.joex.scheduler.{Context, Task}
+import docspell.joex.scheduler.Task
 import docspell.store.queries.QItem
-import docspell.store.records.{RItem, RJob}
+import docspell.store.records.RItem
 
 object ItemHandler {
   def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
     logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ =>
-      deleteByFileIds
+      deleteByFileIds.flatMap(_ => deleteFiles)
     )
 
-  def apply[F[_]: ConcurrentEffect: ContextShift](
+  def newItem[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config
   ): Task[F, ProcessItemArgs, Unit] =
     CreateItem[F]
@@ -31,16 +32,13 @@ object ItemHandler {
         .map(_ => data)
     )
 
-  def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
-    for {
-      current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
-      last = ctx.config.retries == current.getOrElse(0)
-    } yield last
+  def isLastRetry[F[_]: Sync]: Task[F, ProcessItemArgs, Boolean] =
+    Task(_.isLastRetry)
 
   def safeProcess[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config
   )(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
-    Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
+    isLastRetry[F].flatMap {
       case true =>
         ProcessItem[F](cfg)(data).attempt.flatMap({
           case Right(d) =>
@@ -64,6 +62,15 @@ object ItemHandler {
       } yield ()
     }
 
+  private def deleteFiles[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
+    Task(ctx =>
+      Stream
+        .emits(ctx.args.files.map(_.fileMetaId.id))
+        .flatMap(id => ctx.store.bitpeace.delete(id).attempt.drain)
+        .compile
+        .drain
+    )
+
   private def logWarn[F[_]](msg: => String): Task[F, ProcessItemArgs, Unit] =
     Task(_.logger.warn(msg))
 }

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -259,7 +259,7 @@ object ScanMailboxTask {
           priority = Priority.Low,
           tracker = None
         )
-        res <- upload.submit(data, ctx.args.account, false)
+        res <- upload.submit(data, ctx.args.account, false, None)
       } yield res
     }
 

modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala

@@ -1,7 +1,7 @@
 package docspell.joex.scheduler
 
-import cats.Functor
-import cats.effect.{Blocker, Concurrent}
+import cats.{Applicative, Functor}
+import cats.effect._
 import cats.implicits._
 import docspell.common._
 import docspell.store.Store
@@ -23,6 +23,12 @@ trait Context[F[_], A] { self =>
 
   def store: Store[F]
 
+  final def isLastRetry(implicit ev: Applicative[F]): F[Boolean] =
+    for {
+      current <- store.transact(RJob.getRetries(jobId))
+      last = config.retries == current.getOrElse(0)
+    } yield last
+
   def blocker: Blocker
 
   def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] =