Add a folder-id to item processing

This allows to define a folder when uploading files. All generated
items are associated to this folder on creation.

modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala

@@ -58,6 +58,7 @@ object OUpload {
   case class UploadMeta(
       direction: Option[Direction],
       sourceAbbrev: String,
+      folderId: Option[Ident],
       validFileTypes: Seq[MimeType]
   )
 
@@ -123,6 +124,7 @@ object OUpload {
             lang.getOrElse(Language.German),
             data.meta.direction,
             data.meta.sourceAbbrev,
+            data.meta.folderId,
             data.meta.validFileTypes
           )
           args =
@@ -147,7 +149,10 @@ object OUpload {
         (for {
           src <- OptionT(store.transact(RSource.find(sourceId)))
           updata = data.copy(
-            meta = data.meta.copy(sourceAbbrev = src.abbrev),
+            meta = data.meta.copy(
+              sourceAbbrev = src.abbrev,
+              folderId = data.meta.folderId.orElse(src.folderId)
+            ),
             priority = src.priority
           )
           accId = AccountId(src.cid, src.sid)

modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala

@@ -36,6 +36,7 @@ object ProcessItemArgs {
       language: Language,
       direction: Option[Direction],
       sourceAbbrev: String,
+      folderId: Option[Ident],
       validFileTypes: Seq[MimeType]
   )
 

modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

@@ -27,7 +27,9 @@ case class ScanMailboxArgs(
     // delete the after submitting (only if targetFolder is None)
     deleteMail: Boolean,
     // set the direction when submitting
-    direction: Option[Direction]
+    direction: Option[Direction],
+    // set a folder for items
+    itemFolder: Option[Ident]
 )
 
 object ScanMailboxArgs {

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -84,6 +84,7 @@ object JoexAppImpl {
       joex    <- OJoex(client, store)
       upload  <- OUpload(store, queue, cfg.files, joex)
       fts     <- createFtsClient(cfg)(httpClient)
+      itemOps <- OItem(store, fts)
       javaEmil =
         JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))
       sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
@@ -91,7 +92,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             ProcessItemArgs.taskName,
-            ItemHandler.newItem[F](cfg, fts),
+            ItemHandler.newItem[F](cfg, itemOps, fts),
             ItemHandler.onCancel[F]
           )
         )

modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala

@@ -5,6 +5,7 @@ import cats.effect._
 import cats.implicits._
 import fs2.Stream
 
+import docspell.backend.ops.OItem
 import docspell.common.{ItemState, ProcessItemArgs}
 import docspell.ftsclient.FtsClient
 import docspell.joex.Config
@@ -27,11 +28,12 @@ object ItemHandler {
 
   def newItem[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config,
+      itemOps: OItem[F],
       fts: FtsClient[F]
   ): Task[F, Args, Unit] =
     CreateItem[F]
       .flatMap(itemStateTask(ItemState.Processing))
-      .flatMap(safeProcess[F](cfg, fts))
+      .flatMap(safeProcess[F](cfg, itemOps, fts))
       .map(_ => ())
 
   def itemStateTask[F[_]: Sync, A](
@@ -48,11 +50,12 @@ object ItemHandler {
 
   def safeProcess[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config,
+      itemOps: OItem[F],
       fts: FtsClient[F]
   )(data: ItemData): Task[F, Args, ItemData] =
     isLastRetry[F].flatMap {
       case true =>
-        ProcessItem[F](cfg, fts)(data).attempt.flatMap({
+        ProcessItem[F](cfg, itemOps, fts)(data).attempt.flatMap({
           case Right(d) =>
             Task.pure(d)
           case Left(ex) =>
@@ -62,7 +65,7 @@ object ItemHandler {
               .andThen(_ => Sync[F].raiseError(ex))
         })
       case false =>
-        ProcessItem[F](cfg, fts)(data).flatMap(itemStateTask(ItemState.Created))
+        ProcessItem[F](cfg, itemOps, fts)(data).flatMap(itemStateTask(ItemState.Created))
     }
 
   private def markItemCreated[F[_]: Sync]: Task[F, Args, Boolean] =

modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala

@@ -2,6 +2,7 @@ package docspell.joex.process
 
 import cats.effect._
 
+import docspell.backend.ops.OItem
 import docspell.common.ProcessItemArgs
 import docspell.ftsclient.FtsClient
 import docspell.joex.Config
@@ -11,6 +12,7 @@ object ProcessItem {
 
   def apply[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config,
+      itemOps: OItem[F],
       fts: FtsClient[F]
   )(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
     ExtractArchive(item)
@@ -22,6 +24,7 @@ object ProcessItem {
       .flatMap(analysisOnly[F](cfg))
       .flatMap(Task.setProgress(80))
       .flatMap(LinkProposal[F])
+      .flatMap(SetGivenData[F](itemOps))
       .flatMap(Task.setProgress(99))
 
   def analysisOnly[F[_]: Sync](

modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala

@@ -0,0 +1,35 @@
+package docspell.joex.process
+
+import cats.effect._
+import cats.implicits._
+
+import docspell.backend.ops.OItem
+import docspell.common._
+import docspell.joex.scheduler.Task
+
+object SetGivenData {
+
+  def apply[F[_]: Sync](
+      ops: OItem[F]
+  )(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
+    if (data.item.state.isValid)
+      Task
+        .log[F, ProcessItemArgs](_.debug(s"Not setting data on existing item"))
+        .map(_ => data)
+    else
+      Task { ctx =>
+        val itemId     = data.item.id
+        val folderId   = ctx.args.meta.folderId
+        val collective = ctx.args.meta.collective
+        for {
+          _ <- ctx.logger.info("Starting setting given data")
+          _ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'")
+          e <- ops.setFolder(itemId, folderId, collective).attempt
+          _ <- e.fold(
+            ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"),
+            _ => ().pure[F]
+          )
+        } yield data
+      }
+
+}

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -143,7 +143,7 @@ object ScanMailboxTask {
         folder  <- requireFolder(a)(name)
         search  <- searchMails(a)(folder)
         headers <- Kleisli.liftF(filterMessageIds(search.mails))
-        _       <- headers.traverse(handleOne(a, upload))
+        _       <- headers.traverse(handleOne(ctx.args, a, upload))
       } yield ScanResult(name, search.mails.size, search.count - search.mails.size)
 
     def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] =
@@ -239,7 +239,9 @@ object ScanMailboxTask {
           MailOp.pure(())
       }
 
-    def submitMail(upload: OUpload[F])(mail: Mail[F]): F[OUpload.UploadResult] = {
+    def submitMail(upload: OUpload[F], args: Args)(
+        mail: Mail[F]
+    ): F[OUpload.UploadResult] = {
       val file = OUpload.File(
         Some(mail.header.subject + ".eml"),
         Some(MimeType.emls.head),
@@ -251,6 +253,7 @@ object ScanMailboxTask {
         meta = OUpload.UploadMeta(
           Some(dir),
           s"mailbox-${ctx.args.account.user.id}",
+          args.itemFolder,
           Seq.empty
         )
         data = OUpload.UploadData(
@@ -264,14 +267,14 @@ object ScanMailboxTask {
       } yield res
     }
 
-    def handleOne[C](a: Access[F, C], upload: OUpload[F])(
+    def handleOne[C](args: Args, a: Access[F, C], upload: OUpload[F])(
         mh: MailHeader
     ): MailOp[F, C, Unit] =
       for {
         mail <- a.loadMail(mh)
         res <- mail match {
           case Some(m) =>
-            Kleisli.liftF(submitMail(upload)(m).attempt)
+            Kleisli.liftF(submitMail(upload, args)(m).attempt)
           case None =>
             MailOp.pure[F, C, Either[Throwable, OUpload.UploadResult]](
               Either.left(new Exception(s"Mail not found"))

modules/microsite/docs/doc/uploading.md

@@ -144,6 +144,7 @@ structure:
 ```
 { multiple: Bool
 , direction: Maybe String
+, folder: Maybe String
 }
 ```
 
@@ -156,6 +157,11 @@ Furthermore, the direction of the document (one of `incoming` or
 `outgoing`) can be given. It is optional, it can be left out or
 `null`.
 
+A `folder` id can be specified. Each item created by this request will
+be placed into this folder. Errors are logged (for example, the folder
+may have been deleted before the task is executed) and the item is
+then not put into any folder.
+
 This kind of request is very common and most programming languages
 have support for this. For example, here is another curl command
 uploading two files with meta data:

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -2694,6 +2694,13 @@ components:
             The direction to apply to items resulting from importing
             mails. If not set, the value is guessed based on the from
             and to mail headers and your address book.
+        itemFolder:
+          type: string
+          format: ident
+          description: |
+            The folder id that is applied to items resulting from
+            importing mails. If the folder id is not valid when the
+            task executes, items have no folder set.
     ImapSettingsList:
       description: |
         A list of user email settings.
@@ -3437,9 +3444,15 @@ components:
         Meta information for an item upload. The user can specify some
         structured information with a binary file.
 
-        Additional metadata is not required. However, you have to
+        Additional metadata is not required. However, if there is some
-        specifiy whether the corresponding files should become one
+        specified, you have to specifiy whether the corresponding
-        single item or if an item is created for each file.
+        files should become one single item or if an item is created
+        for each file.
+
+        A direction can be given, `Incoming` is used if not specified.
+
+        A folderId can be given, the item is placed into this folder
+        after creation.
       required:
         - multiple
       properties:
@@ -3449,6 +3462,9 @@ components:
         direction:
           type: string
           format: direction
+        folder:
+          type: string
+          format: ident
     Collective:
       description: |
         Information about a collective.
@@ -3519,6 +3535,9 @@ components:
         priority:
           type: string
           format: priority
+        folder:
+          type: string
+          format: ident
         created:
           description: DateTime
           type: integer

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -287,9 +287,11 @@ trait Conversions {
       .find(_.name.exists(_.equalsIgnoreCase("meta")))
       .map(p => parseMeta(p.body))
       .map(fm =>
-        fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))
+        fm.map(m =>
+          (m.multiple, UploadMeta(m.direction, "webapp", m.folder, validFileTypes))
+        )
       )
-      .getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
+      .getOrElse((true, UploadMeta(None, "webapp", None, validFileTypes)).pure[F])
 
     val files = mp.parts
       .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
@@ -491,12 +493,21 @@ trait Conversions {
   // sources
 
   def mkSource(s: RSource): Source =
-    Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
+    Source(
+      s.sid,
+      s.abbrev,
+      s.description,
+      s.counter,
+      s.enabled,
+      s.priority,
+      s.folderId,
+      s.created
+    )
 
   def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
     timeId.map({
       case (id, now) =>
-        RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
+        RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder)
     })
 
   def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
@@ -508,7 +519,8 @@ trait Conversions {
       s.counter,
       s.enabled,
       s.priority,
-      s.created
+      s.created,
+      s.folder
     )
 
   // equipment

modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala

@@ -112,7 +112,8 @@ object ScanMailboxRoutes {
           settings.receivedSinceHours.map(_.toLong).map(Duration.hours),
           settings.targetFolder,
           settings.deleteMail,
-          settings.direction
+          settings.direction,
+          settings.itemFolder
         )
       )
     )
@@ -139,6 +140,7 @@ object ScanMailboxRoutes {
       task.args.receivedSince.map(_.hours.toInt),
       task.args.targetFolder,
       task.args.deleteMail,
-      task.args.direction
+      task.args.direction,
+      task.args.itemFolder
     )
 }

modules/store/src/main/scala/docspell/store/queries/QFolder.scala

@@ -50,6 +50,7 @@ object QFolder {
     def tryDelete =
       for {
         _ <- RItem.removeFolder(id)
+        _ <- RSource.removeFolder(id)
         _ <- RFolderMember.deleteAll(id)
         _ <- RFolder.delete(id)
       } yield FolderChangeResult.success

modules/store/src/main/scala/docspell/store/records/RSource.scala

@@ -15,7 +15,8 @@ case class RSource(
     counter: Int,
     enabled: Boolean,
     priority: Priority,
-    created: Timestamp
+    created: Timestamp,
+    folderId: Option[Ident]
 ) {}
 
 object RSource {
@@ -32,8 +33,10 @@ object RSource {
     val enabled     = Column("enabled")
     val priority    = Column("priority")
     val created     = Column("created")
+    val folder      = Column("folder_id")
 
-    val all = List(sid, cid, abbrev, description, counter, enabled, priority, created)
+    val all =
+      List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
   }
 
   import Columns._
@@ -42,7 +45,7 @@ object RSource {
     val sql = insertRow(
       table,
       all,
-      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}"
+      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
     )
     sql.update.run
   }
@@ -56,7 +59,8 @@ object RSource {
         abbrev.setTo(v.abbrev),
         description.setTo(v.description),
         enabled.setTo(v.enabled),
-        priority.setTo(v.priority)
+        priority.setTo(v.priority),
+        folder.setTo(v.folderId)
       )
     )
     sql.update.run
@@ -97,4 +101,9 @@ object RSource {
 
   def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
     deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run
+
+  def removeFolder(folderId: Ident): ConnectionIO[Int] = {
+    val empty: Option[Ident] = None
+    updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run
+  }
 }