From 5034e12becd2eca41b2a27f3d2002fb434d44bca Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 13 Nov 2020 20:59:03 +0100 Subject: [PATCH 1/2] Add a subject filter to scan-mailbox args --- .../docspell/common/ScanMailboxArgs.scala | 4 +++- .../joex/scanmailbox/ScanMailboxTask.scala | 22 ++++++++++++++++++- .../src/main/resources/docspell-openapi.yml | 7 +++++- .../restserver/routes/ScanMailboxRoutes.scala | 8 ++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index fc73d616..781a3589 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -33,7 +33,9 @@ case class ScanMailboxArgs( // set a filter for files when importing archives fileFilter: Option[Glob], // set a list of tags to apply to new item - tags: Option[List[String]] + tags: Option[List[String]], + // a glob filter for the mail subject + subjectFilter: Option[Glob] ) object ScanMailboxArgs { diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 7a746b9f..3478dd17 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -142,7 +142,7 @@ object ScanMailboxTask { _ <- Kleisli.liftF(ctx.logger.info(s"Processing folder $name")) folder <- requireFolder(a)(name) search <- searchMails(a)(folder) - headers <- Kleisli.liftF(filterMessageIds(search.mails)) + headers <- Kleisli.liftF(filterSubjects(search.mails).flatMap(filterMessageIds)) _ <- headers.traverse(handleOne(ctx.args, a, upload)) } yield ScanResult(name, search.mails.size, search.count - search.mails.size) @@ -178,6 +178,26 @@ object ScanMailboxTask { } yield mails } + def filterSubjects(headers: Vector[MailHeader]): F[Vector[MailHeader]] = + ctx.args.subjectFilter match { + case Some(sf) => + def check(mh: MailHeader): F[Option[MailHeader]] = + if (sf.matches(mh.subject)) + ctx.logger.debug( + s"Including mail '${mh.subject}', it matches the filter." + ) *> Option(mh).pure[F] + else + ctx.logger.debug( + s"Excluding mail '${mh.subject}', it doesn't match the filter." + ) *> (None: Option[MailHeader]).pure[F] + ctx.logger.info( + s"Filtering mails on subject using filter: ${sf.asString}" + ) *> headers.traverseFilter(check) + + case None => + ctx.logger.debug("Not matching on subjects. No filter given") *> headers.pure[F] + } + def filterMessageIds(headers: Vector[MailHeader]): F[Vector[MailHeader]] = NonEmptyList.fromFoldable(headers.flatMap(_.messageId)) match { case Some(nl) => diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 70816521..3b2ba5cd 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3503,7 +3503,12 @@ components: $ref: "#/components/schemas/StringList" fileFilter: description: | - A glob to filter attachments to import. + A glob to filter attachments to import by file name. + type: string + format: glob + subjectFilter: + description: | + A glob to filter attachments to import by subject. type: string format: glob diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 983f491d..36ffbd1d 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -115,7 +115,8 @@ object ScanMailboxRoutes { settings.direction, settings.itemFolder, settings.fileFilter, - settings.tags.map(_.items) + settings.tags.map(_.items), + settings.subjectFilter ) ) ) @@ -137,7 +138,7 @@ object ScanMailboxRoutes { task.id, task.enabled, conn.getOrElse(Ident.unsafe("")), - task.args.folders, //folders + task.args.folders, task.timer, task.args.receivedSince.map(_.hours.toInt), task.args.targetFolder, @@ -145,6 +146,7 @@ object ScanMailboxRoutes { task.args.direction, task.args.itemFolder, task.args.tags.map(StringList.apply), - task.args.fileFilter + task.args.fileFilter, + task.args.subjectFilter ) } From ec5b822e099418864b85dd4aa2b3d759ef51aca9 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 13 Nov 2020 21:35:31 +0100 Subject: [PATCH 2/2] Add subject-filter to scan-mailbox-form --- .../src/main/elm/Comp/ScanMailboxForm.elm | 119 ++++++++++++------ 1 file changed, 79 insertions(+), 40 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm index 2a4544f8..2a72e531 100644 --- a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm @@ -63,6 +63,7 @@ type alias Model = , tagModel : Comp.Dropdown.Model Tag , existingTags : List String , fileFilter : Maybe String + , subjectFilter : Maybe String } @@ -94,6 +95,7 @@ type Msg | GetTagResp (Result Http.Error TagList) | TagDropdownMsg (Comp.Dropdown.Msg Tag) | SetFileFilter String + | SetSubjectFilter String initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg ) @@ -135,6 +137,7 @@ initWith flags s = Maybe.map .items s.tags |> Maybe.withDefault [] , fileFilter = s.fileFilter + , subjectFilter = s.subjectFilter } , Cmd.batch [ Api.getImapSettings flags "" ConnResp @@ -184,6 +187,7 @@ init flags = , tagModel = Util.Tag.makeDropdownModel , existingTags = [] , fileFilter = Nothing + , subjectFilter = Nothing } , Cmd.batch [ Api.getImapSettings flags "" ConnResp @@ -228,6 +232,7 @@ makeSettings model = , schedule = Data.CalEvent.makeEvent timer , itemFolder = model.itemFolderId , fileFilter = model.fileFilter + , subjectFilter = model.subjectFilter , tags = case Comp.Dropdown.getSelected model.tagModel of [] -> @@ -586,6 +591,12 @@ update flags msg model = , Cmd.none ) + SetSubjectFilter str -> + ( { model | subjectFilter = Util.Maybe.fromString str } + , NoAction + , Cmd.none + ) + --- View @@ -653,13 +664,6 @@ view extraClasses settings model = [ text "The folders to go through" ] ] - , Html.map ReceivedHoursMsg - (Comp.IntField.viewWithInfo - "Select mails newer than `now - receivedHours`" - model.receivedHours - "field" - model.receivedHoursModel - ) , div [ class "field" ] [ label [] [ text "Target folder" ] , input @@ -688,6 +692,74 @@ view extraClasses settings model = , text " is not set." ] ] + , div [ class "ui dividing header" ] + [ text "Filter" + ] + , Html.map ReceivedHoursMsg + (Comp.IntField.viewWithInfo + "Select mails newer than `now - receivedHours`" + model.receivedHours + "field" + model.receivedHoursModel + ) + , div + [ class "field" + ] + [ label [] [ text "File Filter" ] + , input + [ type_ "text" + , onInput SetFileFilter + , placeholder "File Filter" + , model.fileFilter + |> Maybe.withDefault "" + |> value + ] + [] + , div [ class "small-info" ] + [ text "Specify a file glob to filter attachments. For example, to only extract pdf files: " + , code [] + [ text "*.pdf" + ] + , text ". If you want to include the mail body, allow html files or " + , code [] + [ text "mail.html" + ] + , text ". Globs can be combined via OR, like this: " + , code [] + [ text "*.pdf|mail.html" + ] + , text ". No file filter defaults to " + , code [] + [ text "*" + ] + , text " that includes all" + ] + ] + , div + [ class "field" + ] + [ label [] [ text "Subject Filter" ] + , input + [ type_ "text" + , onInput SetSubjectFilter + , placeholder "Subject Filter" + , model.subjectFilter + |> Maybe.withDefault "" + |> value + ] + [] + , div [ class "small-info" ] + [ text "Specify a file glob to filter mails by subject. For example: " + , code [] + [ text "*Scanned Document*" + ] + , text ". No file filter defaults to " + , code [] + [ text "*" + ] + , text " that includes all" + ] + ] , div [ class "ui dividing header" ] [ text "Metadata" ] @@ -763,39 +835,6 @@ disappear then. [ text "Choose tags that should be applied to items." ] ] - , div - [ class "field" - ] - [ label [] [ text "File Filter" ] - , input - [ type_ "text" - , onInput SetFileFilter - , placeholder "File Filter" - , model.fileFilter - |> Maybe.withDefault "" - |> value - ] - [] - , div [ class "small-info" ] - [ text "Specify a file glob to filter attachments. For example, to only extract pdf files: " - , code [] - [ text "*.pdf" - ] - , text ". If you want to include the mail body, allow html files or " - , code [] - [ text "mail.html" - ] - , text ". Globs can be combined via OR, like this: " - , code [] - [ text "*.pdf|mail.html" - ] - , text "No file filter defaults to " - , code [] - [ text "*" - ] - , text " that includes all" - ] - ] , div [ class "ui dividing header" ] [ text "Schedule" ]