diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index df9fa87d..65cd8367 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -37,7 +37,9 @@ case class ScanMailboxArgs( // a glob filter for the mail subject subjectFilter: Option[Glob], // the language for extraction and analysis - language: Option[Language] + language: Option[Language], + // apply additional filter to all mails or only imported + postHandleAll: Option[Boolean] ) object ScanMailboxArgs { diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 71bf3902..acb91c00 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -14,6 +14,7 @@ import docspell.joex.scheduler.{Context, Task} import docspell.store.queries.QOrganization import docspell.store.records._ +import _root_.io.circe.syntax._ import emil.SearchQuery.{All, ReceivedDate} import emil.javamail.syntax._ import emil.{MimeType => _, _} @@ -31,8 +32,9 @@ object ScanMailboxTask { Task { ctx => for { _ <- ctx.logger.info( - s"Start importing mails for user ${ctx.args.account.user.id}" + s"=== Start importing mails for user ${ctx.args.account.user.id}" ) + _ <- ctx.logger.debug(s"Settings: ${ctx.args.asJson.noSpaces}") mailCfg <- getMailSettings(ctx) folders = ctx.args.folders.mkString(", ") userId = ctx.args.account.user @@ -249,25 +251,29 @@ object ScanMailboxTask { .getOrElse(Direction.Incoming) } - def postHandle[C](a: Access[F, C])(mh: MailHeader): MailOp[F, C, Unit] = + def postHandle[C](a: Access[F, C])(mh: MailHeaderItem): MailOp[F, C, Unit] = { + val postHandleAll = ctx.args.postHandleAll.exists(identity) ctx.args.targetFolder match { - case Some(tf) => - logOp(_.debug(s"Post handling mail: ${mh.subject} - moving to folder: $tf")) + case Some(tf) if postHandleAll || mh.process => + logOp( + _.debug(s"Post handling mail: ${mh.mh.subject} - moving to folder: $tf") + ) .flatMap(_ => - a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh, folder)) + a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh.mh, folder)) ) - case None if ctx.args.deleteMail => - logOp(_.debug(s"Post handling mail: ${mh.subject} - deleting mail.")).flatMap( - _ => - a.deleteMail(mh).flatMapF { r => + case None if ctx.args.deleteMail && (postHandleAll || mh.process) => + logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - deleting mail.")) + .flatMap(_ => + a.deleteMail(mh.mh).flatMapF { r => if (r.count == 0) ctx.logger.warn(s"Mail could not be deleted!") else ().pure[F] } - ) - case None => - logOp(_.debug(s"Post handling mail: ${mh.subject} - no handling defined!")) + ) + case _ => + logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - no handling defined!")) } + } def submitMail(upload: OUpload[F], args: Args)( mail: Mail[F] @@ -321,7 +327,7 @@ object ScanMailboxTask { Kleisli.liftF( ctx.logger.warn(s"Error submitting '${mh.mh.subject}': ${ex.getMessage}") ), - _ => postHandle(a)(mh.mh) + _ => postHandle(a)(mh) ) } yield () } diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 90ce21ae..494824a6 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3904,6 +3904,8 @@ components: processing mails. type: string format: language + postHandleAll: + type: boolean ImapSettingsList: description: | diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 596fd3f6..1eec5f2a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -117,7 +117,8 @@ object ScanMailboxRoutes { settings.fileFilter, settings.tags.map(_.items), settings.subjectFilter, - settings.language + settings.language, + settings.postHandleAll ) ) ) @@ -149,6 +150,7 @@ object ScanMailboxRoutes { task.args.tags.map(StringList.apply), task.args.fileFilter, task.args.subjectFilter, - task.args.language + task.args.language, + task.args.postHandleAll ) } diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm index 68cd45bc..ef2ae954 100644 --- a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm @@ -68,6 +68,8 @@ type alias Model = , subjectFilter : Maybe String , languageModel : Comp.FixedDropdown.Model Language , language : Maybe Language + , postHandleAll : Bool + , menuTab : MenuTab } @@ -79,6 +81,15 @@ type Action | NoAction +type MenuTab + = TabGeneral + | TabProcessing + | TabAdditionalFilter + | TabPostProcessing + | TabMetadata + | TabSchedule + + type Msg = Submit | Cancel @@ -102,6 +113,8 @@ type Msg | SetSubjectFilter String | LanguageMsg (Comp.FixedDropdown.Msg Language) | RemoveLanguage + | TogglePostHandleAll + | SetMenuTab MenuTab initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg ) @@ -147,6 +160,8 @@ initWith flags s = , languageModel = Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all) , language = Maybe.andThen Data.Language.fromString s.language + , postHandleAll = Maybe.withDefault False s.postHandleAll + , menuTab = TabGeneral } , Cmd.batch [ Api.getImapSettings flags "" ConnResp @@ -200,6 +215,8 @@ init flags = , languageModel = Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all) , language = Nothing + , postHandleAll = False + , menuTab = TabGeneral } , Cmd.batch [ Api.getImapSettings flags "" ConnResp @@ -232,7 +249,7 @@ makeSettings model = infolders = if model.folders == [] then - Invalid [ "No folders given" ] [] + Invalid [ "No processing folders given" ] [] else Valid model.folders @@ -260,6 +277,7 @@ makeSettings model = |> StringList |> Just , language = Maybe.map Data.Language.toIso3 model.language + , postHandleAll = Just model.postHandleAll } in Data.Validated.map3 make @@ -646,6 +664,18 @@ update flags msg model = , Cmd.none ) + TogglePostHandleAll -> + ( { model | postHandleAll = not model.postHandleAll } + , NoAction + , Cmd.none + ) + + SetMenuTab tab -> + ( { model | menuTab = tab } + , NoAction + , Cmd.none + ) + --- View @@ -674,267 +704,27 @@ view extraClasses settings model = , ( "success", isFormSuccess model ) ] ] - [ Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete) - , div - [ classList - [ ( "ui dimmer", True ) - , ( "active", model.loading > 0 ) - ] - ] - [ div [ class "ui text loader" ] - [ text "Loading..." - ] - ] - , div [ class "inline field" ] - [ div [ class "ui checkbox" ] - [ input - [ type_ "checkbox" - , onCheck (\_ -> ToggleEnabled) - , checked model.enabled - ] - [] - , label [] [ text "Enabled" ] - ] - , span [ class "small-info" ] - [ text "Enable or disable this task." - ] - ] - , div [ class "required field" ] - [ label [] [ text "Mailbox" ] - , Html.map ConnMsg (Comp.Dropdown.view settings model.connectionModel) - , span [ class "small-info" ] - [ text "The IMAP connection to use when sending notification mails." - ] - ] - , div [ class "required field" ] - [ label [] [ text "Folders" ] - , Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel) - , span [ class "small-info" ] - [ text "The folders to go through" - ] - ] - , div [ class "field" ] - [ label [] [ text "Target folder" ] - , input - [ type_ "text" - , onInput SetTargetFolder - , Maybe.withDefault "" model.targetFolder |> value - ] - [] - , span [ class "small-info" ] - [ text "Move all mails successfully submitted into this folder." - ] - ] - , div [ class "inline field" ] - [ div [ class "ui checkbox" ] - [ input - [ type_ "checkbox" - , onCheck (\_ -> ToggleDeleteMail) - , checked model.deleteMail - ] - [] - , label [] [ text "Delete imported mails" ] - ] - , span [ class "small-info" ] - [ text "Whether to delete all mails successfully imported into docspell. This only applies if " - , em [] [ text "target folder" ] - , text " is not set." - ] - ] - , div [ class "ui dividing header" ] - [ text "Filter" - ] - , Html.map ReceivedHoursMsg - (Comp.IntField.viewWithInfo - "Select mails newer than `now - receivedHours`" - model.receivedHours - "field" - model.receivedHoursModel + [ viewMenu model + , div [ class "ui bottom attached segment" ] + (case model.menuTab of + TabGeneral -> + viewGeneral settings model + + TabProcessing -> + viewProcessing model + + TabAdditionalFilter -> + viewAdditionalFilter model + + TabPostProcessing -> + viewPostProcessing model + + TabMetadata -> + viewMetadata settings model + + TabSchedule -> + viewSchedule model ) - , div - [ class "field" - ] - [ label [] [ text "File Filter" ] - , input - [ type_ "text" - , onInput SetFileFilter - , placeholder "File Filter" - , model.fileFilter - |> Maybe.withDefault "" - |> value - ] - [] - , div [ class "small-info" ] - [ text "Specify a file glob to filter attachments. For example, to only extract pdf files: " - , code [] - [ text "*.pdf" - ] - , text ". If you want to include the mail body, allow html files or " - , code [] - [ text "mail.html" - ] - , text ". Globs can be combined via OR, like this: " - , code [] - [ text "*.pdf|mail.html" - ] - , text ". No file filter defaults to " - , code [] - [ text "*" - ] - , text " that includes all" - ] - ] - , div - [ class "field" - ] - [ label [] [ text "Subject Filter" ] - , input - [ type_ "text" - , onInput SetSubjectFilter - , placeholder "Subject Filter" - , model.subjectFilter - |> Maybe.withDefault "" - |> value - ] - [] - , div [ class "small-info" ] - [ text "Specify a file glob to filter mails by subject. For example: " - , code [] - [ text "*Scanned Document*" - ] - , text ". No file filter defaults to " - , code [] - [ text "*" - ] - , text " that includes all" - ] - ] - , div [ class "ui dividing header" ] - [ text "Metadata" - ] - , div [ class "required field" ] - [ label [] [ text "Item direction" ] - , div [ class "grouped fields" ] - [ div [ class "field" ] - [ div [ class "ui radio checkbox" ] - [ input - [ type_ "radio" - , checked (model.direction == Nothing) - , onCheck (\_ -> DirectionMsg Nothing) - ] - [] - , label [] [ text "Automatic" ] - ] - ] - , div [ class "field" ] - [ div [ class "ui radio checkbox" ] - [ input - [ type_ "radio" - , checked (model.direction == Just Incoming) - , onCheck (\_ -> DirectionMsg (Just Incoming)) - ] - [] - , label [] [ text "Incoming" ] - ] - ] - , div [ class "field" ] - [ div [ class "ui radio checkbox" ] - [ input - [ type_ "radio" - , checked (model.direction == Just Outgoing) - , onCheck (\_ -> DirectionMsg (Just Outgoing)) - ] - [] - , label [] [ text "Outgoing" ] - ] - ] - , span [ class "small-info" ] - [ text "Sets the direction for an item. If you know all mails are incoming or " - , text "outgoing, you can set it here. Otherwise it will be guessed from looking " - , text "at sender and receiver." - ] - ] - ] - , div [ class "field" ] - [ label [] - [ text "Item Folder" - ] - , Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel) - , span [ class "small-info" ] - [ text "Put all items from this mailbox into the selected folder" - ] - , div - [ classList - [ ( "ui warning message", True ) - , ( "hidden", isFolderMember model ) - ] - ] - [ Markdown.toHtml [] """ -You are **not a member** of this folder. Items created from mails in -this mailbox will be **hidden** from any search results. Use a folder -where you are a member of to make items visible. This message will -disappear then. - """ - ] - ] - , div [ class "field" ] - [ label [] [ text "Tags" ] - , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel) - , div [ class "small-info" ] - [ text "Choose tags that should be applied to items." - ] - ] - , div [ class "field" ] - [ label [] - [ text "Language" - ] - , div [ class "ui action input" ] - [ Html.map LanguageMsg - (Comp.FixedDropdown.viewStyled "fluid" - (Maybe.map mkLanguageItem model.language) - model.languageModel - ) - , a - [ class "ui icon button" - , href "#" - , onClick RemoveLanguage - ] - [ i [ class "delete icon" ] [] - ] - ] - , div [ class "small-info" ] - [ text "Used for text extraction and text analysis. The " - , text "collective's default language is used, if not specified here." - ] - ] - , div [ class "ui dividing header" ] - [ text "Schedule" - ] - , div [ class "required field" ] - [ label [] - [ text "Schedule" - , a - [ class "right-float" - , href "https://github.com/eikek/calev#what-are-calendar-events" - , target "_blank" - ] - [ i [ class "help icon" ] [] - , text "Click here for help" - ] - ] - , Html.map CalEventMsg - (Comp.CalEventInput.view "" - (Data.Validated.value model.schedule) - model.scheduleModel - ) - , span [ class "small-info" ] - [ text "Specify how often and when this task should run. " - , text "Use English 3-letter weekdays. Either a single value, " - , text "a list (ex. 1,2,3), a range (ex. 1..3) or a '*' (meaning all) " - , text "is allowed for each part." - ] - ] - , div [ class "ui divider" ] [] , div [ classList [ ( "ui message", True ) @@ -974,9 +764,348 @@ disappear then. ] [ text "Start Once" ] + , Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete) + , div + [ classList + [ ( "ui dimmer", True ) + , ( "active", model.loading > 0 ) + , ( "invisible hidden", model.loading == 0 ) + ] + ] + [ div [ class "ui text loader" ] + [ text "Loading..." + ] + ] ] +viewMenu : Model -> Html Msg +viewMenu model = + let + tabLink tab txt = + a + [ class "item" + , classList + [ ( "active", model.menuTab == tab ) ] + , href "#" + , onClick (SetMenuTab tab) + ] + [ text txt + ] + in + div [ class "ui top attached stacked tabular menu" ] + [ tabLink TabGeneral "General" + , tabLink TabProcessing "Processing" + , tabLink TabAdditionalFilter "Additional Filter" + , tabLink TabPostProcessing "Post Processing" + , tabLink TabMetadata "Metadata" + , tabLink TabSchedule "Schedule" + ] + + +viewGeneral : UiSettings -> Model -> List (Html Msg) +viewGeneral settings model = + [ div [ class "inline field" ] + [ div [ class "ui checkbox" ] + [ input + [ type_ "checkbox" + , onCheck (\_ -> ToggleEnabled) + , checked model.enabled + ] + [] + , label [] [ text "Enabled" ] + ] + , span [ class "small-info" ] + [ text "Enable or disable this task." + ] + ] + , div [ class "required field" ] + [ label [] [ text "Mailbox" ] + , Html.map ConnMsg (Comp.Dropdown.view settings model.connectionModel) + , span [ class "small-info" ] + [ text "The IMAP connection to use when sending notification mails." + ] + ] + ] + + +viewProcessing : Model -> List (Html Msg) +viewProcessing model = + [ div [ class "ui message" ] + [ text "These settings define which mails are fetched from the mail server." + ] + , div [ class "required field" ] + [ label [] [ text "Folders" ] + , Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel) + , span [ class "small-info" ] + [ text "The folders to go through" + ] + ] + , Html.map ReceivedHoursMsg + (Comp.IntField.viewWithInfo + "Select mails newer than `now - receivedHours`" + model.receivedHours + "field" + model.receivedHoursModel + ) + ] + + +viewAdditionalFilter : Model -> List (Html Msg) +viewAdditionalFilter model = + [ div [ class "ui message" ] + [ text "These filters are applied to mails that have been fetched from the " + , text "mailbox to select those that should be imported." + ] + , div + [ class "field" + ] + [ label [] [ text "File Filter" ] + , input + [ type_ "text" + , onInput SetFileFilter + , placeholder "File Filter" + , model.fileFilter + |> Maybe.withDefault "" + |> value + ] + [] + , div [ class "small-info" ] + [ text "Specify a file glob to filter attachments. For example, to only extract pdf files: " + , code [] + [ text "*.pdf" + ] + , text ". If you want to include the mail body, allow html files or " + , code [] + [ text "mail.html" + ] + , text ". Globs can be combined via OR, like this: " + , code [] + [ text "*.pdf|mail.html" + ] + , text ". No file filter defaults to " + , code [] + [ text "*" + ] + , text " that includes all" + ] + ] + , div + [ class "field" + ] + [ label [] [ text "Subject Filter" ] + , input + [ type_ "text" + , onInput SetSubjectFilter + , placeholder "Subject Filter" + , model.subjectFilter + |> Maybe.withDefault "" + |> value + ] + [] + , div [ class "small-info" ] + [ text "Specify a file glob to filter mails by subject. For example: " + , code [] + [ text "*Scanned Document*" + ] + , text ". No file filter defaults to " + , code [] + [ text "*" + ] + , text " that includes all" + ] + ] + ] + + +viewPostProcessing : Model -> List (Html Msg) +viewPostProcessing model = + [ div [ class "ui message" ] + [ text "This defines what happens to mails that have been downloaded." + ] + , div [ class "inline field" ] + [ div [ class "ui checkbox" ] + [ input + [ type_ "checkbox" + , onCheck (\_ -> TogglePostHandleAll) + , checked model.postHandleAll + ] + [] + , label [] [ text "Apply post-processing to all fetched mails." ] + ] + , span [ class "small-info" ] + [ text "When mails are not imported due to the 'Additional Filters', this flag can " + , text "control whether they should be moved to a target folder or deleted (whatever is " + , text "defined here) nevertheless. If unchecked only imported mails " + , text "are post-processed, others stay where they are." + ] + ] + , div [ class "field" ] + [ label [] [ text "Target folder" ] + , input + [ type_ "text" + , onInput SetTargetFolder + , Maybe.withDefault "" model.targetFolder |> value + ] + [] + , span [ class "small-info" ] + [ text "Move mails into this folder." + ] + ] + , div [ class "inline field" ] + [ div [ class "ui checkbox" ] + [ input + [ type_ "checkbox" + , onCheck (\_ -> ToggleDeleteMail) + , checked model.deleteMail + ] + [] + , label [] [ text "Delete imported mails" ] + ] + , span [ class "small-info" ] + [ text "Whether to delete all mails fetched by docspell. This only applies if " + , em [] [ text "target folder" ] + , text " is not set." + ] + ] + ] + + +viewMetadata : UiSettings -> Model -> List (Html Msg) +viewMetadata settings model = + [ div [ class "ui message" ] + [ text "Define metadata that should be attached to all items created by this task." + ] + , div [ class "required field" ] + [ label [] [ text "Item direction" ] + , div [ class "grouped fields" ] + [ div [ class "field" ] + [ div [ class "ui radio checkbox" ] + [ input + [ type_ "radio" + , checked (model.direction == Nothing) + , onCheck (\_ -> DirectionMsg Nothing) + ] + [] + , label [] [ text "Automatic" ] + ] + ] + , div [ class "field" ] + [ div [ class "ui radio checkbox" ] + [ input + [ type_ "radio" + , checked (model.direction == Just Incoming) + , onCheck (\_ -> DirectionMsg (Just Incoming)) + ] + [] + , label [] [ text "Incoming" ] + ] + ] + , div [ class "field" ] + [ div [ class "ui radio checkbox" ] + [ input + [ type_ "radio" + , checked (model.direction == Just Outgoing) + , onCheck (\_ -> DirectionMsg (Just Outgoing)) + ] + [] + , label [] [ text "Outgoing" ] + ] + ] + , span [ class "small-info" ] + [ text "Sets the direction for an item. If you know all mails are incoming or " + , text "outgoing, you can set it here. Otherwise it will be guessed from looking " + , text "at sender and receiver." + ] + ] + ] + , div [ class "field" ] + [ label [] + [ text "Item Folder" + ] + , Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel) + , span [ class "small-info" ] + [ text "Put all items from this mailbox into the selected folder" + ] + , div + [ classList + [ ( "ui warning message", True ) + , ( "hidden", isFolderMember model ) + ] + ] + [ Markdown.toHtml [] """ +You are **not a member** of this folder. Items created from mails in +this mailbox will be **hidden** from any search results. Use a folder +where you are a member of to make items visible. This message will +disappear then. + """ + ] + ] + , div [ class "field" ] + [ label [] [ text "Tags" ] + , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel) + , div [ class "small-info" ] + [ text "Choose tags that should be applied to items." + ] + ] + , div [ class "field" ] + [ label [] + [ text "Language" + ] + , div [ class "ui action input" ] + [ Html.map LanguageMsg + (Comp.FixedDropdown.viewStyled "fluid" + (Maybe.map mkLanguageItem model.language) + model.languageModel + ) + , a + [ class "ui icon button" + , href "#" + , onClick RemoveLanguage + ] + [ i [ class "delete icon" ] [] + ] + ] + , div [ class "small-info" ] + [ text "Used for text extraction and text analysis. The " + , text "collective's default language is used, if not specified here." + ] + ] + ] + + +viewSchedule : Model -> List (Html Msg) +viewSchedule model = + [ div [ class "ui message" ] + [ text "Define when mails should be imported." + ] + , div [ class "required field" ] + [ label [] + [ text "Schedule" + , a + [ class "right-float" + , href "https://github.com/eikek/calev#what-are-calendar-events" + , target "_blank" + ] + [ i [ class "help icon" ] [] + , text "Click here for help" + ] + ] + , Html.map CalEventMsg + (Comp.CalEventInput.view "" + (Data.Validated.value model.schedule) + model.scheduleModel + ) + , span [ class "small-info" ] + [ text "Specify how often and when this task should run. " + , text "Use English 3-letter weekdays. Either a single value, " + , text "a list (ex. 1,2,3), a range (ex. 1..3) or a '*' (meaning all) " + , text "is allowed for each part." + ] + ] + ] + + isFolderMember : Model -> Bool isFolderMember model = let diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm index 9bbd854a..ec9b4780 100644 --- a/modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm +++ b/modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm @@ -247,7 +247,7 @@ view settings model = viewForm : UiSettings -> Comp.ScanMailboxForm.Model -> Html Msg viewForm settings model = - Html.map DetailMsg (Comp.ScanMailboxForm.view "segment" settings model) + Html.map DetailMsg (Comp.ScanMailboxForm.view "" settings model) viewList : Model -> Html Msg