From 55a6f7aaf69d3272f06594b45c8889cf76227b40 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 11 Nov 2020 01:08:07 +0100 Subject: [PATCH] Add more properties to upload meta data --- build.sbt | 22 ++++++++-------- .../scala/docspell/backend/ops/OUpload.scala | 8 ++++-- .../docspell/common/ProcessItemArgs.scala | 4 ++- .../docspell/common/ScanMailboxArgs.scala | 6 ++++- .../docspell/joex/process/ReProcessItem.scala | 4 ++- .../joex/process/TextExtraction.scala | 4 ++- .../joex/scanmailbox/ScanMailboxTask.scala | 4 ++- .../src/main/resources/docspell-openapi.yml | 25 +++++++++++++++++++ .../restserver/conv/Conversions.scala | 6 +++-- .../restserver/routes/ScanMailboxRoutes.scala | 8 ++++-- 10 files changed, 70 insertions(+), 21 deletions(-) diff --git a/build.sbt b/build.sbt index 333b7121..e7b0a853 100644 --- a/build.sbt +++ b/build.sbt @@ -124,9 +124,8 @@ val buildInfoSettings = Seq( val openapiScalaSettings = Seq( openapiScalaConfig := ScalaConfig() .withJson(ScalaJson.circeSemiauto) - .addMapping(CustomMapping.forType({ - case TypeDef("LocalDateTime", _) => - TypeDef("Timestamp", Imports("docspell.common.Timestamp")) + .addMapping(CustomMapping.forType({ case TypeDef("LocalDateTime", _) => + TypeDef("Timestamp", Imports("docspell.common.Timestamp")) })) .addMapping(CustomMapping.forFormatType({ case "ident" => @@ -182,6 +181,8 @@ val openapiScalaSettings = Seq( ) ) ) + case "glob" => + field => field.copy(typeDef = TypeDef("Glob", Imports("docspell.common.Glob"))) })) ) @@ -595,11 +596,10 @@ def copyWebjarResources( src.flatMap { dir => if (dir.isDirectory) { val files = (dir ** "*").filter(_.isFile).get.pair(Path.relativeTo(dir)) - files.flatMap { - case (f, name) => - val target = targetDir / name - IO.createDirectories(Seq(target.getParentFile)) - copyWithGZ(f, target) + files.flatMap { case (f, name) => + val target = targetDir / name + IO.createDirectories(Seq(target.getParentFile)) + copyWithGZ(f, target) } } else { val target = targetDir / dir.name @@ -633,11 +633,13 @@ def compileElm( } def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = { - val target = out / "Webjars.scala" + val target = out / "Webjars.scala" val badChars = "-.".toSet val fields = wj .map(m => - s"""val ${m.name.toLowerCase.filter(c => !badChars.contains(c))} = "/${m.name}/${m.revision}" """ + s"""val ${m.name.toLowerCase.filter(c => + !badChars.contains(c) + )} = "/${m.name}/${m.revision}" """ ) .mkString("\n\n") val content = s"""package docspell.restserver.webapp diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index e71a131f..8723ed40 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -60,7 +60,9 @@ object OUpload { sourceAbbrev: String, folderId: Option[Ident], validFileTypes: Seq[MimeType], - skipDuplicates: Boolean + skipDuplicates: Boolean, + fileFilter: Option[Glob], + tags: List[String] ) case class UploadData[F[_]]( @@ -127,7 +129,9 @@ object OUpload { data.meta.sourceAbbrev, data.meta.folderId, data.meta.validFileTypes, - data.meta.skipDuplicates + data.meta.skipDuplicates, + data.meta.fileFilter, + data.meta.tags.some ) args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index 6e5427be..aba6974e 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -38,7 +38,9 @@ object ProcessItemArgs { sourceAbbrev: String, folderId: Option[Ident], validFileTypes: Seq[MimeType], - skipDuplicate: Boolean + skipDuplicate: Boolean, + fileFilter: Option[Glob], + tags: Option[List[String]] ) object ProcessMeta { diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index fa86b903..fc73d616 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -29,7 +29,11 @@ case class ScanMailboxArgs( // set the direction when submitting direction: Option[Direction], // set a folder for items - itemFolder: Option[Ident] + itemFolder: Option[Ident], + // set a filter for files when importing archives + fileFilter: Option[Glob], + // set a list of tags to apply to new item + tags: Option[List[String]] ) object ScanMailboxArgs { diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index dd7747db..07fb2901 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -91,7 +91,9 @@ object ReProcessItem { "", //source-id None, //folder Seq.empty, - false + false, + None, + None ), Nil ).pure[F] diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index 112034a4..db2988b8 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -47,7 +47,9 @@ object TextExtraction { _ <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)).toSeq: _*) dur <- start _ <- ctx.logger.info(s"Text extraction finished in ${dur.formatExact}") - } yield item.copy(metas = txt.map(_.am), tags = txt.flatMap(_.tags).distinct.toList) + } yield item + .copy(metas = txt.map(_.am)) + .appendTags(txt.flatMap(_.tags).distinct.toList) } // -- helpers diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 0fee001a..a01b1676 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -255,7 +255,9 @@ object ScanMailboxTask { s"mailbox-${ctx.args.account.user.id}", args.itemFolder, Seq.empty, - true + true, + args.fileFilter, + args.tags.getOrElse(Nil) ) data = OUpload.UploadData( multiple = false, diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index fbba6e89..3f2c4131 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3499,6 +3499,14 @@ components: The folder id that is applied to items resulting from importing mails. If the folder id is not valid when the task executes, items have no folder set. + tags: + $ref: "#/components/schemas/StringList" + fileFilter: + description: | + A glob to filter attachments to import. + type: string + format: glob + ImapSettingsList: description: | A list of user email settings. @@ -4256,6 +4264,18 @@ components: A folderId can be given, the item is placed into this folder after creation. + The `fileFilter` is an optional glob for filtering files to + import. Only applicable if archive files are uploaded. It + applies to all of them. For example, to only import pdf files + when uploading e-mails, use `*.pdf`. If the pattern doesn't + contain a slash `/`, then it is applied to all file names. + Otherwise it is applied to the complete path in the archive + (useful for zip files). Note that the archive file itself is + always saved completely, too. + + The `tags` input allows to provide tags that should be applied + to the item being created. This only works if the tags already + exist. It is possible to specify their ids or names. required: - multiple properties: @@ -4271,6 +4291,11 @@ components: skipDuplicates: type: boolean default: false + tags: + $ref: "#/components/schemas/StringList" + fileFilter: + type: string + format: glob Collective: description: | diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index aba61555..e5191e07 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -310,13 +310,15 @@ trait Conversions { sourceName, m.folder, validFileTypes, - m.skipDuplicates.getOrElse(false) + m.skipDuplicates.getOrElse(false), + m.fileFilter, + m.tags.map(_.items).getOrElse(Nil) ) ) ) ) .getOrElse( - (true, UploadMeta(None, sourceName, None, validFileTypes, false)).pure[F] + (true, UploadMeta(None, sourceName, None, validFileTypes, false, None, Nil)).pure[F] ) val files = mp.parts diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 4a1a738c..983f491d 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -113,7 +113,9 @@ object ScanMailboxRoutes { settings.targetFolder, settings.deleteMail, settings.direction, - settings.itemFolder + settings.itemFolder, + settings.fileFilter, + settings.tags.map(_.items) ) ) ) @@ -141,6 +143,8 @@ object ScanMailboxRoutes { task.args.targetFolder, task.args.deleteMail, task.args.direction, - task.args.itemFolder + task.args.itemFolder, + task.args.tags.map(StringList.apply), + task.args.fileFilter ) }