Refactor scan mailbox form and add flag for post-processing

Mails are filtered once by using an imap search and then by some globs
to filter files and subjects. Imap can search by subject via a
string-contains, but not via globs or patterns (afaik). The subject
filter is applied to all downloaded mail headers. Now for post
processing (moving to some target folder or deleting), it can be
chosen to post-process all "seen" mails or only those that matched all
filters.

modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

@@ -37,7 +37,9 @@ case class ScanMailboxArgs(
     // a glob filter for the mail subject
     subjectFilter: Option[Glob],
     // the language for extraction and analysis
-    language: Option[Language]
+    language: Option[Language],
+    // apply additional filter to all mails or only imported
+    postHandleAll: Option[Boolean]
 )
 
 object ScanMailboxArgs {

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -14,6 +14,7 @@ import docspell.joex.scheduler.{Context, Task}
 import docspell.store.queries.QOrganization
 import docspell.store.records._
 
+import _root_.io.circe.syntax._
 import emil.SearchQuery.{All, ReceivedDate}
 import emil.javamail.syntax._
 import emil.{MimeType => _, _}
@@ -31,8 +32,9 @@ object ScanMailboxTask {
     Task { ctx =>
       for {
         _ <- ctx.logger.info(
-          s"Start importing mails for user ${ctx.args.account.user.id}"
+          s"=== Start importing mails for user ${ctx.args.account.user.id}"
         )
+        _       <- ctx.logger.debug(s"Settings: ${ctx.args.asJson.noSpaces}")
         mailCfg <- getMailSettings(ctx)
         folders  = ctx.args.folders.mkString(", ")
         userId   = ctx.args.account.user
@@ -249,24 +251,28 @@ object ScanMailboxTask {
         .getOrElse(Direction.Incoming)
     }
 
-    def postHandle[C](a: Access[F, C])(mh: MailHeader): MailOp[F, C, Unit] =
+    def postHandle[C](a: Access[F, C])(mh: MailHeaderItem): MailOp[F, C, Unit] = {
+      val postHandleAll = ctx.args.postHandleAll.exists(identity)
       ctx.args.targetFolder match {
-        case Some(tf) =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - moving to folder: $tf"))
-            .flatMap(_ =>
-              a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh, folder))
+        case Some(tf) if postHandleAll || mh.process =>
+          logOp(
+            _.debug(s"Post handling mail: ${mh.mh.subject} - moving to folder: $tf")
           )
-        case None if ctx.args.deleteMail =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - deleting mail.")).flatMap(
-            _ =>
-              a.deleteMail(mh).flatMapF { r =>
+            .flatMap(_ =>
+              a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh.mh, folder))
+            )
+        case None if ctx.args.deleteMail && (postHandleAll || mh.process) =>
+          logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - deleting mail."))
+            .flatMap(_ =>
+              a.deleteMail(mh.mh).flatMapF { r =>
                 if (r.count == 0)
                   ctx.logger.warn(s"Mail could not be deleted!")
                 else ().pure[F]
               }
             )
-        case None =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - no handling defined!"))
+        case _ =>
+          logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - no handling defined!"))
+      }
     }
 
     def submitMail(upload: OUpload[F], args: Args)(
@@ -321,7 +327,7 @@ object ScanMailboxTask {
             Kleisli.liftF(
               ctx.logger.warn(s"Error submitting '${mh.mh.subject}': ${ex.getMessage}")
             ),
-          _ => postHandle(a)(mh.mh)
+          _ => postHandle(a)(mh)
         )
       } yield ()
   }

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -3904,6 +3904,8 @@ components:
             processing mails.
           type: string
           format: language
+        postHandleAll:
+          type: boolean
 
     ImapSettingsList:
       description: |

modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala

@@ -117,7 +117,8 @@ object ScanMailboxRoutes {
           settings.fileFilter,
           settings.tags.map(_.items),
           settings.subjectFilter,
-          settings.language
+          settings.language,
+          settings.postHandleAll
         )
       )
     )
@@ -149,6 +150,7 @@ object ScanMailboxRoutes {
       task.args.tags.map(StringList.apply),
       task.args.fileFilter,
       task.args.subjectFilter,
-      task.args.language
+      task.args.language,
+      task.args.postHandleAll
     )
 }

modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm

@@ -68,6 +68,8 @@ type alias Model =
     , subjectFilter : Maybe String
     , languageModel : Comp.FixedDropdown.Model Language
     , language : Maybe Language
+    , postHandleAll : Bool
+    , menuTab : MenuTab
     }
 
 
@@ -79,6 +81,15 @@ type Action
     | NoAction
 
 
+type MenuTab
+    = TabGeneral
+    | TabProcessing
+    | TabAdditionalFilter
+    | TabPostProcessing
+    | TabMetadata
+    | TabSchedule
+
+
 type Msg
     = Submit
     | Cancel
@@ -102,6 +113,8 @@ type Msg
     | SetSubjectFilter String
     | LanguageMsg (Comp.FixedDropdown.Msg Language)
     | RemoveLanguage
+    | TogglePostHandleAll
+    | SetMenuTab MenuTab
 
 
 initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg )
@@ -147,6 +160,8 @@ initWith flags s =
         , languageModel =
             Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
         , language = Maybe.andThen Data.Language.fromString s.language
+        , postHandleAll = Maybe.withDefault False s.postHandleAll
+        , menuTab = TabGeneral
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
@@ -200,6 +215,8 @@ init flags =
       , languageModel =
             Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
       , language = Nothing
+      , postHandleAll = False
+      , menuTab = TabGeneral
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
@@ -232,7 +249,7 @@ makeSettings model =
 
         infolders =
             if model.folders == [] then
-                Invalid [ "No folders given" ] []
+                Invalid [ "No processing folders given" ] []
 
             else
                 Valid model.folders
@@ -260,6 +277,7 @@ makeSettings model =
                                 |> StringList
                                 |> Just
                 , language = Maybe.map Data.Language.toIso3 model.language
+                , postHandleAll = Just model.postHandleAll
             }
     in
     Data.Validated.map3 make
@@ -646,6 +664,18 @@ update flags msg model =
             , Cmd.none
             )
 
+        TogglePostHandleAll ->
+            ( { model | postHandleAll = not model.postHandleAll }
+            , NoAction
+            , Cmd.none
+            )
+
+        SetMenuTab tab ->
+            ( { model | menuTab = tab }
+            , NoAction
+            , Cmd.none
+            )
+
 
 
 --- View
@@ -674,18 +704,108 @@ view extraClasses settings model =
             , ( "success", isFormSuccess model )
             ]
         ]
-        [ Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
+        [ viewMenu model
+        , div [ class "ui bottom attached segment" ]
+            (case model.menuTab of
+                TabGeneral ->
+                    viewGeneral settings model
+
+                TabProcessing ->
+                    viewProcessing model
+
+                TabAdditionalFilter ->
+                    viewAdditionalFilter model
+
+                TabPostProcessing ->
+                    viewPostProcessing model
+
+                TabMetadata ->
+                    viewMetadata settings model
+
+                TabSchedule ->
+                    viewSchedule model
+            )
+        , div
+            [ classList
+                [ ( "ui message", True )
+                , ( "success", isFormSuccess model )
+                , ( "error", isFormError model )
+                , ( "hidden", model.formMsg == Nothing )
+                ]
+            ]
+            [ Maybe.map .message model.formMsg
+                |> Maybe.withDefault ""
+                |> text
+            ]
+        , button
+            [ class "ui primary button"
+            , onClick Submit
+            ]
+            [ text "Submit"
+            ]
+        , button
+            [ class "ui secondary button"
+            , onClick Cancel
+            ]
+            [ text "Cancel"
+            ]
+        , button
+            [ classList
+                [ ( "ui red button", True )
+                , ( "hidden invisible", model.settings.id == "" )
+                ]
+            , onClick RequestDelete
+            ]
+            [ text "Delete"
+            ]
+        , button
+            [ class "ui right floated button"
+            , onClick StartOnce
+            ]
+            [ text "Start Once"
+            ]
+        , Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
         , div
             [ classList
                 [ ( "ui dimmer", True )
                 , ( "active", model.loading > 0 )
+                , ( "invisible hidden", model.loading == 0 )
                 ]
             ]
             [ div [ class "ui text loader" ]
                 [ text "Loading..."
                 ]
             ]
-        , div [ class "inline field" ]
+        ]
+
+
+viewMenu : Model -> Html Msg
+viewMenu model =
+    let
+        tabLink tab txt =
+            a
+                [ class "item"
+                , classList
+                    [ ( "active", model.menuTab == tab ) ]
+                , href "#"
+                , onClick (SetMenuTab tab)
+                ]
+                [ text txt
+                ]
+    in
+    div [ class "ui top attached stacked tabular menu" ]
+        [ tabLink TabGeneral "General"
+        , tabLink TabProcessing "Processing"
+        , tabLink TabAdditionalFilter "Additional Filter"
+        , tabLink TabPostProcessing "Post Processing"
+        , tabLink TabMetadata "Metadata"
+        , tabLink TabSchedule "Schedule"
+        ]
+
+
+viewGeneral : UiSettings -> Model -> List (Html Msg)
+viewGeneral settings model =
+    [ div [ class "inline field" ]
         [ div [ class "ui checkbox" ]
             [ input
                 [ type_ "checkbox"
@@ -706,6 +826,14 @@ view extraClasses settings model =
             [ text "The IMAP connection to use when sending notification mails."
             ]
         ]
+    ]
+
+
+viewProcessing : Model -> List (Html Msg)
+viewProcessing model =
+    [ div [ class "ui message" ]
+        [ text "These settings define which mails are fetched from the mail server."
+        ]
     , div [ class "required field" ]
         [ label [] [ text "Folders" ]
         , Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel)
@@ -713,37 +841,6 @@ view extraClasses settings model =
             [ text "The folders to go through"
             ]
         ]
-        , div [ class "field" ]
-            [ label [] [ text "Target folder" ]
-            , input
-                [ type_ "text"
-                , onInput SetTargetFolder
-                , Maybe.withDefault "" model.targetFolder |> value
-                ]
-                []
-            , span [ class "small-info" ]
-                [ text "Move all mails successfully submitted into this folder."
-                ]
-            ]
-        , div [ class "inline field" ]
-            [ div [ class "ui checkbox" ]
-                [ input
-                    [ type_ "checkbox"
-                    , onCheck (\_ -> ToggleDeleteMail)
-                    , checked model.deleteMail
-                    ]
-                    []
-                , label [] [ text "Delete imported mails" ]
-                ]
-            , span [ class "small-info" ]
-                [ text "Whether to delete all mails successfully imported into docspell. This only applies if "
-                , em [] [ text "target folder" ]
-                , text " is not set."
-                ]
-            ]
-        , div [ class "ui dividing header" ]
-            [ text "Filter"
-            ]
     , Html.map ReceivedHoursMsg
         (Comp.IntField.viewWithInfo
             "Select mails newer than `now - receivedHours`"
@@ -751,6 +848,15 @@ view extraClasses settings model =
             "field"
             model.receivedHoursModel
         )
+    ]
+
+
+viewAdditionalFilter : Model -> List (Html Msg)
+viewAdditionalFilter model =
+    [ div [ class "ui message" ]
+        [ text "These filters are applied to mails that have been fetched from the "
+        , text "mailbox to select those that should be imported."
+        ]
     , div
         [ class "field"
         ]
@@ -809,8 +915,66 @@ view extraClasses settings model =
             , text " that includes all"
             ]
         ]
-        , div [ class "ui dividing header" ]
-            [ text "Metadata"
+    ]
+
+
+viewPostProcessing : Model -> List (Html Msg)
+viewPostProcessing model =
+    [ div [ class "ui message" ]
+        [ text "This defines what happens to mails that have been downloaded."
+        ]
+    , div [ class "inline field" ]
+        [ div [ class "ui checkbox" ]
+            [ input
+                [ type_ "checkbox"
+                , onCheck (\_ -> TogglePostHandleAll)
+                , checked model.postHandleAll
+                ]
+                []
+            , label [] [ text "Apply post-processing to all fetched mails." ]
+            ]
+        , span [ class "small-info" ]
+            [ text "When mails are not imported due to the 'Additional Filters', this flag can "
+            , text "control whether they should be moved to a target folder or deleted (whatever is "
+            , text "defined here) nevertheless. If unchecked only imported mails "
+            , text "are post-processed, others stay where they are."
+            ]
+        ]
+    , div [ class "field" ]
+        [ label [] [ text "Target folder" ]
+        , input
+            [ type_ "text"
+            , onInput SetTargetFolder
+            , Maybe.withDefault "" model.targetFolder |> value
+            ]
+            []
+        , span [ class "small-info" ]
+            [ text "Move mails into this folder."
+            ]
+        ]
+    , div [ class "inline field" ]
+        [ div [ class "ui checkbox" ]
+            [ input
+                [ type_ "checkbox"
+                , onCheck (\_ -> ToggleDeleteMail)
+                , checked model.deleteMail
+                ]
+                []
+            , label [] [ text "Delete imported mails" ]
+            ]
+        , span [ class "small-info" ]
+            [ text "Whether to delete all mails fetched by docspell. This only applies if "
+            , em [] [ text "target folder" ]
+            , text " is not set."
+            ]
+        ]
+    ]
+
+
+viewMetadata : UiSettings -> Model -> List (Html Msg)
+viewMetadata settings model =
+    [ div [ class "ui message" ]
+        [ text "Define metadata that should be attached to all items created by this task."
         ]
     , div [ class "required field" ]
         [ label [] [ text "Item direction" ]
@@ -907,8 +1071,13 @@ disappear then.
             , text "collective's default language is used, if not specified here."
             ]
         ]
-        , div [ class "ui dividing header" ]
-            [ text "Schedule"
+    ]
+
+
+viewSchedule : Model -> List (Html Msg)
+viewSchedule model =
+    [ div [ class "ui message" ]
+        [ text "Define when mails should be imported."
         ]
     , div [ class "required field" ]
         [ label []
@@ -934,46 +1103,6 @@ disappear then.
             , text "is allowed for each part."
             ]
         ]
-        , div [ class "ui divider" ] []
-        , div
-            [ classList
-                [ ( "ui message", True )
-                , ( "success", isFormSuccess model )
-                , ( "error", isFormError model )
-                , ( "hidden", model.formMsg == Nothing )
-                ]
-            ]
-            [ Maybe.map .message model.formMsg
-                |> Maybe.withDefault ""
-                |> text
-            ]
-        , button
-            [ class "ui primary button"
-            , onClick Submit
-            ]
-            [ text "Submit"
-            ]
-        , button
-            [ class "ui secondary button"
-            , onClick Cancel
-            ]
-            [ text "Cancel"
-            ]
-        , button
-            [ classList
-                [ ( "ui red button", True )
-                , ( "hidden invisible", model.settings.id == "" )
-                ]
-            , onClick RequestDelete
-            ]
-            [ text "Delete"
-            ]
-        , button
-            [ class "ui right floated button"
-            , onClick StartOnce
-            ]
-            [ text "Start Once"
-            ]
     ]
 
 

modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm

@@ -247,7 +247,7 @@ view settings model =
 
 viewForm : UiSettings -> Comp.ScanMailboxForm.Model -> Html Msg
 viewForm settings model =
-    Html.map DetailMsg (Comp.ScanMailboxForm.view "segment" settings model)
+    Html.map DetailMsg (Comp.ScanMailboxForm.view "" settings model)
 
 
 viewList : Model -> Html Msg