From 5b01c93711c7e15087bffa4ce0fb3fe3cc6b860c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 14 Jul 2020 21:25:44 +0200 Subject: [PATCH] Add a folder-id to item processing This allows to define a folder when uploading files. All generated items are associated to this folder on creation. --- .../scala/docspell/backend/ops/OUpload.scala | 7 +++- .../docspell/common/ProcessItemArgs.scala | 1 + .../docspell/common/ScanMailboxArgs.scala | 4 ++- .../scala/docspell/joex/JoexAppImpl.scala | 3 +- .../docspell/joex/process/ItemHandler.scala | 9 +++-- .../docspell/joex/process/ProcessItem.scala | 3 ++ .../docspell/joex/process/SetGivenData.scala | 35 +++++++++++++++++++ .../joex/scanmailbox/ScanMailboxTask.scala | 11 +++--- modules/microsite/docs/doc/uploading.md | 6 ++++ .../src/main/resources/docspell-openapi.yml | 25 +++++++++++-- .../restserver/conv/Conversions.scala | 22 +++++++++--- .../restserver/routes/ScanMailboxRoutes.scala | 6 ++-- .../docspell/store/queries/QFolder.scala | 1 + .../docspell/store/records/RSource.scala | 17 ++++++--- 14 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index 6d9f0669..7c4b043c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -58,6 +58,7 @@ object OUpload { case class UploadMeta( direction: Option[Direction], sourceAbbrev: String, + folderId: Option[Ident], validFileTypes: Seq[MimeType] ) @@ -123,6 +124,7 @@ object OUpload { lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, + data.meta.folderId, data.meta.validFileTypes ) args = @@ -147,7 +149,10 @@ object OUpload { (for { src <- OptionT(store.transact(RSource.find(sourceId))) updata = data.copy( - meta = data.meta.copy(sourceAbbrev = src.abbrev), + meta = data.meta.copy( + sourceAbbrev = src.abbrev, + folderId = data.meta.folderId.orElse(src.folderId) + ), priority = src.priority ) accId = AccountId(src.cid, src.sid) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index a4b209dd..9e3faf2b 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -36,6 +36,7 @@ object ProcessItemArgs { language: Language, direction: Option[Direction], sourceAbbrev: String, + folderId: Option[Ident], validFileTypes: Seq[MimeType] ) diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index c4687a41..fa86b903 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -27,7 +27,9 @@ case class ScanMailboxArgs( // delete the after submitting (only if targetFolder is None) deleteMail: Boolean, // set the direction when submitting - direction: Option[Direction] + direction: Option[Direction], + // set a folder for items + itemFolder: Option[Ident] ) object ScanMailboxArgs { diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index ee59f2f9..965659b7 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -84,6 +84,7 @@ object JoexAppImpl { joex <- OJoex(client, store) upload <- OUpload(store, queue, cfg.files, joex) fts <- createFtsClient(cfg)(httpClient) + itemOps <- OItem(store, fts) javaEmil = JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug)) sch <- SchedulerBuilder(cfg.scheduler, blocker, store) @@ -91,7 +92,7 @@ object JoexAppImpl { .withTask( JobTask.json( ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, fts), + ItemHandler.newItem[F](cfg, itemOps, fts), ItemHandler.onCancel[F] ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index f18f29e7..4da8f779 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -5,6 +5,7 @@ import cats.effect._ import cats.implicits._ import fs2.Stream +import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} import docspell.ftsclient.FtsClient import docspell.joex.Config @@ -27,11 +28,12 @@ object ItemHandler { def newItem[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, + itemOps: OItem[F], fts: FtsClient[F] ): Task[F, Args, Unit] = CreateItem[F] .flatMap(itemStateTask(ItemState.Processing)) - .flatMap(safeProcess[F](cfg, fts)) + .flatMap(safeProcess[F](cfg, itemOps, fts)) .map(_ => ()) def itemStateTask[F[_]: Sync, A]( @@ -48,11 +50,12 @@ object ItemHandler { def safeProcess[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, + itemOps: OItem[F], fts: FtsClient[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - ProcessItem[F](cfg, fts)(data).attempt.flatMap({ + ProcessItem[F](cfg, itemOps, fts)(data).attempt.flatMap({ case Right(d) => Task.pure(d) case Left(ex) => @@ -62,7 +65,7 @@ object ItemHandler { .andThen(_ => Sync[F].raiseError(ex)) }) case false => - ProcessItem[F](cfg, fts)(data).flatMap(itemStateTask(ItemState.Created)) + ProcessItem[F](cfg, itemOps, fts)(data).flatMap(itemStateTask(ItemState.Created)) } private def markItemCreated[F[_]: Sync]: Task[F, Args, Boolean] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index c81230bf..139ec8f6 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -2,6 +2,7 @@ package docspell.joex.process import cats.effect._ +import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs import docspell.ftsclient.FtsClient import docspell.joex.Config @@ -11,6 +12,7 @@ object ProcessItem { def apply[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, + itemOps: OItem[F], fts: FtsClient[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = ExtractArchive(item) @@ -22,6 +24,7 @@ object ProcessItem { .flatMap(analysisOnly[F](cfg)) .flatMap(Task.setProgress(80)) .flatMap(LinkProposal[F]) + .flatMap(SetGivenData[F](itemOps)) .flatMap(Task.setProgress(99)) def analysisOnly[F[_]: Sync]( diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala new file mode 100644 index 00000000..ba51af23 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -0,0 +1,35 @@ +package docspell.joex.process + +import cats.effect._ +import cats.implicits._ + +import docspell.backend.ops.OItem +import docspell.common._ +import docspell.joex.scheduler.Task + +object SetGivenData { + + def apply[F[_]: Sync]( + ops: OItem[F] + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = + if (data.item.state.isValid) + Task + .log[F, ProcessItemArgs](_.debug(s"Not setting data on existing item")) + .map(_ => data) + else + Task { ctx => + val itemId = data.item.id + val folderId = ctx.args.meta.folderId + val collective = ctx.args.meta.collective + for { + _ <- ctx.logger.info("Starting setting given data") + _ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'") + e <- ops.setFolder(itemId, folderId, collective).attempt + _ <- e.fold( + ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"), + _ => ().pure[F] + ) + } yield data + } + +} diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 7c0ef6bb..e98ef3ea 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -143,7 +143,7 @@ object ScanMailboxTask { folder <- requireFolder(a)(name) search <- searchMails(a)(folder) headers <- Kleisli.liftF(filterMessageIds(search.mails)) - _ <- headers.traverse(handleOne(a, upload)) + _ <- headers.traverse(handleOne(ctx.args, a, upload)) } yield ScanResult(name, search.mails.size, search.count - search.mails.size) def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] = @@ -239,7 +239,9 @@ object ScanMailboxTask { MailOp.pure(()) } - def submitMail(upload: OUpload[F])(mail: Mail[F]): F[OUpload.UploadResult] = { + def submitMail(upload: OUpload[F], args: Args)( + mail: Mail[F] + ): F[OUpload.UploadResult] = { val file = OUpload.File( Some(mail.header.subject + ".eml"), Some(MimeType.emls.head), @@ -251,6 +253,7 @@ object ScanMailboxTask { meta = OUpload.UploadMeta( Some(dir), s"mailbox-${ctx.args.account.user.id}", + args.itemFolder, Seq.empty ) data = OUpload.UploadData( @@ -264,14 +267,14 @@ object ScanMailboxTask { } yield res } - def handleOne[C](a: Access[F, C], upload: OUpload[F])( + def handleOne[C](args: Args, a: Access[F, C], upload: OUpload[F])( mh: MailHeader ): MailOp[F, C, Unit] = for { mail <- a.loadMail(mh) res <- mail match { case Some(m) => - Kleisli.liftF(submitMail(upload)(m).attempt) + Kleisli.liftF(submitMail(upload, args)(m).attempt) case None => MailOp.pure[F, C, Either[Throwable, OUpload.UploadResult]]( Either.left(new Exception(s"Mail not found")) diff --git a/modules/microsite/docs/doc/uploading.md b/modules/microsite/docs/doc/uploading.md index 5233a4f3..bd2d45d9 100644 --- a/modules/microsite/docs/doc/uploading.md +++ b/modules/microsite/docs/doc/uploading.md @@ -144,6 +144,7 @@ structure: ``` { multiple: Bool , direction: Maybe String +, folder: Maybe String } ``` @@ -156,6 +157,11 @@ Furthermore, the direction of the document (one of `incoming` or `outgoing`) can be given. It is optional, it can be left out or `null`. +A `folder` id can be specified. Each item created by this request will +be placed into this folder. Errors are logged (for example, the folder +may have been deleted before the task is executed) and the item is +then not put into any folder. + This kind of request is very common and most programming languages have support for this. For example, here is another curl command uploading two files with meta data: diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 5102aa97..02a673cc 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2694,6 +2694,13 @@ components: The direction to apply to items resulting from importing mails. If not set, the value is guessed based on the from and to mail headers and your address book. + itemFolder: + type: string + format: ident + description: | + The folder id that is applied to items resulting from + importing mails. If the folder id is not valid when the + task executes, items have no folder set. ImapSettingsList: description: | A list of user email settings. @@ -3437,9 +3444,15 @@ components: Meta information for an item upload. The user can specify some structured information with a binary file. - Additional metadata is not required. However, you have to - specifiy whether the corresponding files should become one - single item or if an item is created for each file. + Additional metadata is not required. However, if there is some + specified, you have to specifiy whether the corresponding + files should become one single item or if an item is created + for each file. + + A direction can be given, `Incoming` is used if not specified. + + A folderId can be given, the item is placed into this folder + after creation. required: - multiple properties: @@ -3449,6 +3462,9 @@ components: direction: type: string format: direction + folder: + type: string + format: ident Collective: description: | Information about a collective. @@ -3519,6 +3535,9 @@ components: priority: type: string format: priority + folder: + type: string + format: ident created: description: DateTime type: integer diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index fa94c30b..7c57b5e3 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -287,9 +287,11 @@ trait Conversions { .find(_.name.exists(_.equalsIgnoreCase("meta"))) .map(p => parseMeta(p.body)) .map(fm => - fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes))) + fm.map(m => + (m.multiple, UploadMeta(m.direction, "webapp", m.folder, validFileTypes)) + ) ) - .getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) + .getOrElse((true, UploadMeta(None, "webapp", None, validFileTypes)).pure[F]) val files = mp.parts .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))) @@ -491,12 +493,21 @@ trait Conversions { // sources def mkSource(s: RSource): Source = - Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) + Source( + s.sid, + s.abbrev, + s.description, + s.counter, + s.enabled, + s.priority, + s.folderId, + s.created + ) def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = timeId.map({ case (id, now) => - RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) + RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder) }) def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource = @@ -508,7 +519,8 @@ trait Conversions { s.counter, s.enabled, s.priority, - s.created + s.created, + s.folder ) // equipment diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 7e3ab8cc..4a1a738c 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -112,7 +112,8 @@ object ScanMailboxRoutes { settings.receivedSinceHours.map(_.toLong).map(Duration.hours), settings.targetFolder, settings.deleteMail, - settings.direction + settings.direction, + settings.itemFolder ) ) ) @@ -139,6 +140,7 @@ object ScanMailboxRoutes { task.args.receivedSince.map(_.hours.toInt), task.args.targetFolder, task.args.deleteMail, - task.args.direction + task.args.direction, + task.args.itemFolder ) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala index e613f6e9..9c922d48 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala @@ -50,6 +50,7 @@ object QFolder { def tryDelete = for { _ <- RItem.removeFolder(id) + _ <- RSource.removeFolder(id) _ <- RFolderMember.deleteAll(id) _ <- RFolder.delete(id) } yield FolderChangeResult.success diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala index 8e529e95..ea76a919 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -15,7 +15,8 @@ case class RSource( counter: Int, enabled: Boolean, priority: Priority, - created: Timestamp + created: Timestamp, + folderId: Option[Ident] ) {} object RSource { @@ -32,8 +33,10 @@ object RSource { val enabled = Column("enabled") val priority = Column("priority") val created = Column("created") + val folder = Column("folder_id") - val all = List(sid, cid, abbrev, description, counter, enabled, priority, created) + val all = + List(sid, cid, abbrev, description, counter, enabled, priority, created, folder) } import Columns._ @@ -42,7 +45,7 @@ object RSource { val sql = insertRow( table, all, - fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}" + fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}" ) sql.update.run } @@ -56,7 +59,8 @@ object RSource { abbrev.setTo(v.abbrev), description.setTo(v.description), enabled.setTo(v.enabled), - priority.setTo(v.priority) + priority.setTo(v.priority), + folder.setTo(v.folderId) ) ) sql.update.run @@ -97,4 +101,9 @@ object RSource { def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run + + def removeFolder(folderId: Ident): ConnectionIO[Int] = { + val empty: Option[Ident] = None + updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run + } }