Merge pull request #590 from eikek/scan-mailbox-filter

Refactor scan mailbox form and add flag for post-processing

modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

@@ -37,7 +37,9 @@ case class ScanMailboxArgs(
     // a glob filter for the mail subject
     subjectFilter: Option[Glob],
     // the language for extraction and analysis
-    language: Option[Language]
+    language: Option[Language],
+    // apply additional filter to all mails or only imported
+    postHandleAll: Option[Boolean]
 )
 
 object ScanMailboxArgs {

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -14,6 +14,7 @@ import docspell.joex.scheduler.{Context, Task}
 import docspell.store.queries.QOrganization
 import docspell.store.records._
 
+import _root_.io.circe.syntax._
 import emil.SearchQuery.{All, ReceivedDate}
 import emil.javamail.syntax._
 import emil.{MimeType => _, _}
@@ -31,8 +32,9 @@ object ScanMailboxTask {
     Task { ctx =>
       for {
         _ <- ctx.logger.info(
-          s"Start importing mails for user ${ctx.args.account.user.id}"
+          s"=== Start importing mails for user ${ctx.args.account.user.id}"
         )
+        _       <- ctx.logger.debug(s"Settings: ${ctx.args.asJson.noSpaces}")
         mailCfg <- getMailSettings(ctx)
         folders  = ctx.args.folders.mkString(", ")
         userId   = ctx.args.account.user
@@ -249,24 +251,28 @@ object ScanMailboxTask {
         .getOrElse(Direction.Incoming)
     }
 
-    def postHandle[C](a: Access[F, C])(mh: MailHeader): MailOp[F, C, Unit] =
+    def postHandle[C](a: Access[F, C])(mh: MailHeaderItem): MailOp[F, C, Unit] = {
+      val postHandleAll = ctx.args.postHandleAll.exists(identity)
       ctx.args.targetFolder match {
-        case Some(tf) =>
+        case Some(tf) if postHandleAll || mh.process =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - moving to folder: $tf"))
+          logOp(
-            .flatMap(_ =>
+            _.debug(s"Post handling mail: ${mh.mh.subject} - moving to folder: $tf")
-              a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh, folder))
           )
-        case None if ctx.args.deleteMail =>
+            .flatMap(_ =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - deleting mail.")).flatMap(
+              a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh.mh, folder))
-            _ =>
+            )
-              a.deleteMail(mh).flatMapF { r =>
+        case None if ctx.args.deleteMail && (postHandleAll || mh.process) =>
+          logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - deleting mail."))
+            .flatMap(_ =>
+              a.deleteMail(mh.mh).flatMapF { r =>
                 if (r.count == 0)
                   ctx.logger.warn(s"Mail could not be deleted!")
                 else ().pure[F]
               }
             )
-        case None =>
+        case _ =>
-          logOp(_.debug(s"Post handling mail: ${mh.subject} - no handling defined!"))
+          logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - no handling defined!"))
+      }
     }
 
     def submitMail(upload: OUpload[F], args: Args)(
@@ -321,7 +327,7 @@ object ScanMailboxTask {
             Kleisli.liftF(
               ctx.logger.warn(s"Error submitting '${mh.mh.subject}': ${ex.getMessage}")
             ),
-          _ => postHandle(a)(mh.mh)
+          _ => postHandle(a)(mh)
         )
       } yield ()
   }

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -3904,6 +3904,8 @@ components:
             processing mails.
           type: string
           format: language
+        postHandleAll:
+          type: boolean
 
     ImapSettingsList:
       description: |

modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala

@@ -117,7 +117,8 @@ object ScanMailboxRoutes {
           settings.fileFilter,
           settings.tags.map(_.items),
           settings.subjectFilter,
-          settings.language
+          settings.language,
+          settings.postHandleAll
         )
       )
     )
@@ -149,6 +150,7 @@ object ScanMailboxRoutes {
       task.args.tags.map(StringList.apply),
       task.args.fileFilter,
       task.args.subjectFilter,
-      task.args.language
+      task.args.language,
+      task.args.postHandleAll
     )
 }

modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm

@@ -68,6 +68,8 @@ type alias Model =
     , subjectFilter : Maybe String
     , languageModel : Comp.FixedDropdown.Model Language
     , language : Maybe Language
+    , postHandleAll : Bool
+    , menuTab : MenuTab
     }
 
 
@@ -79,6 +81,15 @@ type Action
     | NoAction
 
 
+type MenuTab
+    = TabGeneral
+    | TabProcessing
+    | TabAdditionalFilter
+    | TabPostProcessing
+    | TabMetadata
+    | TabSchedule
+
+
 type Msg
     = Submit
     | Cancel
@@ -102,6 +113,8 @@ type Msg
     | SetSubjectFilter String
     | LanguageMsg (Comp.FixedDropdown.Msg Language)
     | RemoveLanguage
+    | TogglePostHandleAll
+    | SetMenuTab MenuTab
 
 
 initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg )
@@ -147,6 +160,8 @@ initWith flags s =
         , languageModel =
             Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
         , language = Maybe.andThen Data.Language.fromString s.language
+        , postHandleAll = Maybe.withDefault False s.postHandleAll
+        , menuTab = TabGeneral
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
@@ -200,6 +215,8 @@ init flags =
       , languageModel =
             Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
       , language = Nothing
+      , postHandleAll = False
+      , menuTab = TabGeneral
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
@@ -232,7 +249,7 @@ makeSettings model =
 
         infolders =
             if model.folders == [] then
-                Invalid [ "No folders given" ] []
+                Invalid [ "No processing folders given" ] []
 
             else
                 Valid model.folders
@@ -260,6 +277,7 @@ makeSettings model =
                                 |> StringList
                                 |> Just
                 , language = Maybe.map Data.Language.toIso3 model.language
+                , postHandleAll = Just model.postHandleAll
             }
     in
     Data.Validated.map3 make
@@ -646,6 +664,18 @@ update flags msg model =
             , Cmd.none
             )
 
+        TogglePostHandleAll ->
+            ( { model | postHandleAll = not model.postHandleAll }
+            , NoAction
+            , Cmd.none
+            )
+
+        SetMenuTab tab ->
+            ( { model | menuTab = tab }
+            , NoAction
+            , Cmd.none
+            )
+
 
 
 --- View
@@ -674,18 +704,108 @@ view extraClasses settings model =
             , ( "success", isFormSuccess model )
             ]
         ]
-        [ Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
+        [ viewMenu model
+        , div [ class "ui bottom attached segment" ]
+            (case model.menuTab of
+                TabGeneral ->
+                    viewGeneral settings model
+
+                TabProcessing ->
+                    viewProcessing model
+
+                TabAdditionalFilter ->
+                    viewAdditionalFilter model
+
+                TabPostProcessing ->
+                    viewPostProcessing model
+
+                TabMetadata ->
+                    viewMetadata settings model
+
+                TabSchedule ->
+                    viewSchedule model
+            )
+        , div
+            [ classList
+                [ ( "ui message", True )
+                , ( "success", isFormSuccess model )
+                , ( "error", isFormError model )
+                , ( "hidden", model.formMsg == Nothing )
+                ]
+            ]
+            [ Maybe.map .message model.formMsg
+                |> Maybe.withDefault ""
+                |> text
+            ]
+        , button
+            [ class "ui primary button"
+            , onClick Submit
+            ]
+            [ text "Submit"
+            ]
+        , button
+            [ class "ui secondary button"
+            , onClick Cancel
+            ]
+            [ text "Cancel"
+            ]
+        , button
+            [ classList
+                [ ( "ui red button", True )
+                , ( "hidden invisible", model.settings.id == "" )
+                ]
+            , onClick RequestDelete
+            ]
+            [ text "Delete"
+            ]
+        , button
+            [ class "ui right floated button"
+            , onClick StartOnce
+            ]
+            [ text "Start Once"
+            ]
+        , Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
         , div
             [ classList
                 [ ( "ui dimmer", True )
                 , ( "active", model.loading > 0 )
+                , ( "invisible hidden", model.loading == 0 )
                 ]
             ]
             [ div [ class "ui text loader" ]
                 [ text "Loading..."
                 ]
             ]
-        , div [ class "inline field" ]
+        ]
+
+
+viewMenu : Model -> Html Msg
+viewMenu model =
+    let
+        tabLink tab txt =
+            a
+                [ class "item"
+                , classList
+                    [ ( "active", model.menuTab == tab ) ]
+                , href "#"
+                , onClick (SetMenuTab tab)
+                ]
+                [ text txt
+                ]
+    in
+    div [ class "ui top attached stacked tabular menu" ]
+        [ tabLink TabGeneral "General"
+        , tabLink TabProcessing "Processing"
+        , tabLink TabAdditionalFilter "Additional Filter"
+        , tabLink TabPostProcessing "Post Processing"
+        , tabLink TabMetadata "Metadata"
+        , tabLink TabSchedule "Schedule"
+        ]
+
+
+viewGeneral : UiSettings -> Model -> List (Html Msg)
+viewGeneral settings model =
+    [ div [ class "inline field" ]
         [ div [ class "ui checkbox" ]
             [ input
                 [ type_ "checkbox"
@@ -706,6 +826,14 @@ view extraClasses settings model =
             [ text "The IMAP connection to use when sending notification mails."
             ]
         ]
+    ]
+
+
+viewProcessing : Model -> List (Html Msg)
+viewProcessing model =
+    [ div [ class "ui message" ]
+        [ text "These settings define which mails are fetched from the mail server."
+        ]
     , div [ class "required field" ]
         [ label [] [ text "Folders" ]
         , Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel)
@@ -713,37 +841,6 @@ view extraClasses settings model =
             [ text "The folders to go through"
             ]
         ]
-        , div [ class "field" ]
-            [ label [] [ text "Target folder" ]
-            , input
-                [ type_ "text"
-                , onInput SetTargetFolder
-                , Maybe.withDefault "" model.targetFolder |> value
-                ]
-                []
-            , span [ class "small-info" ]
-                [ text "Move all mails successfully submitted into this folder."
-                ]
-            ]
-        , div [ class "inline field" ]
-            [ div [ class "ui checkbox" ]
-                [ input
-                    [ type_ "checkbox"
-                    , onCheck (\_ -> ToggleDeleteMail)
-                    , checked model.deleteMail
-                    ]
-                    []
-                , label [] [ text "Delete imported mails" ]
-                ]
-            , span [ class "small-info" ]
-                [ text "Whether to delete all mails successfully imported into docspell. This only applies if "
-                , em [] [ text "target folder" ]
-                , text " is not set."
-                ]
-            ]
-        , div [ class "ui dividing header" ]
-            [ text "Filter"
-            ]
     , Html.map ReceivedHoursMsg
         (Comp.IntField.viewWithInfo
             "Select mails newer than `now - receivedHours`"
@@ -751,6 +848,15 @@ view extraClasses settings model =
             "field"
             model.receivedHoursModel
         )
+    ]
+
+
+viewAdditionalFilter : Model -> List (Html Msg)
+viewAdditionalFilter model =
+    [ div [ class "ui message" ]
+        [ text "These filters are applied to mails that have been fetched from the "
+        , text "mailbox to select those that should be imported."
+        ]
     , div
         [ class "field"
         ]
@@ -809,8 +915,66 @@ view extraClasses settings model =
             , text " that includes all"
             ]
         ]
-        , div [ class "ui dividing header" ]
+    ]
-            [ text "Metadata"
+
+
+viewPostProcessing : Model -> List (Html Msg)
+viewPostProcessing model =
+    [ div [ class "ui message" ]
+        [ text "This defines what happens to mails that have been downloaded."
+        ]
+    , div [ class "inline field" ]
+        [ div [ class "ui checkbox" ]
+            [ input
+                [ type_ "checkbox"
+                , onCheck (\_ -> TogglePostHandleAll)
+                , checked model.postHandleAll
+                ]
+                []
+            , label [] [ text "Apply post-processing to all fetched mails." ]
+            ]
+        , span [ class "small-info" ]
+            [ text "When mails are not imported due to the 'Additional Filters', this flag can "
+            , text "control whether they should be moved to a target folder or deleted (whatever is "
+            , text "defined here) nevertheless. If unchecked only imported mails "
+            , text "are post-processed, others stay where they are."
+            ]
+        ]
+    , div [ class "field" ]
+        [ label [] [ text "Target folder" ]
+        , input
+            [ type_ "text"
+            , onInput SetTargetFolder
+            , Maybe.withDefault "" model.targetFolder |> value
+            ]
+            []
+        , span [ class "small-info" ]
+            [ text "Move mails into this folder."
+            ]
+        ]
+    , div [ class "inline field" ]
+        [ div [ class "ui checkbox" ]
+            [ input
+                [ type_ "checkbox"
+                , onCheck (\_ -> ToggleDeleteMail)
+                , checked model.deleteMail
+                ]
+                []
+            , label [] [ text "Delete imported mails" ]
+            ]
+        , span [ class "small-info" ]
+            [ text "Whether to delete all mails fetched by docspell. This only applies if "
+            , em [] [ text "target folder" ]
+            , text " is not set."
+            ]
+        ]
+    ]
+
+
+viewMetadata : UiSettings -> Model -> List (Html Msg)
+viewMetadata settings model =
+    [ div [ class "ui message" ]
+        [ text "Define metadata that should be attached to all items created by this task."
         ]
     , div [ class "required field" ]
         [ label [] [ text "Item direction" ]
@@ -907,8 +1071,13 @@ disappear then.
             , text "collective's default language is used, if not specified here."
             ]
         ]
-        , div [ class "ui dividing header" ]
+    ]
-            [ text "Schedule"
+
+
+viewSchedule : Model -> List (Html Msg)
+viewSchedule model =
+    [ div [ class "ui message" ]
+        [ text "Define when mails should be imported."
         ]
     , div [ class "required field" ]
         [ label []
@@ -934,46 +1103,6 @@ disappear then.
             , text "is allowed for each part."
             ]
         ]
-        , div [ class "ui divider" ] []
-        , div
-            [ classList
-                [ ( "ui message", True )
-                , ( "success", isFormSuccess model )
-                , ( "error", isFormError model )
-                , ( "hidden", model.formMsg == Nothing )
-                ]
-            ]
-            [ Maybe.map .message model.formMsg
-                |> Maybe.withDefault ""
-                |> text
-            ]
-        , button
-            [ class "ui primary button"
-            , onClick Submit
-            ]
-            [ text "Submit"
-            ]
-        , button
-            [ class "ui secondary button"
-            , onClick Cancel
-            ]
-            [ text "Cancel"
-            ]
-        , button
-            [ classList
-                [ ( "ui red button", True )
-                , ( "hidden invisible", model.settings.id == "" )
-                ]
-            , onClick RequestDelete
-            ]
-            [ text "Delete"
-            ]
-        , button
-            [ class "ui right floated button"
-            , onClick StartOnce
-            ]
-            [ text "Start Once"
-            ]
     ]
 
 

modules/webapp/src/main/elm/Comp/ScanMailboxManage.elm

@@ -247,7 +247,7 @@ view settings model =
 
 viewForm : UiSettings -> Comp.ScanMailboxForm.Model -> Html Msg
 viewForm settings model =
-    Html.map DetailMsg (Comp.ScanMailboxForm.view "segment" settings model)
+    Html.map DetailMsg (Comp.ScanMailboxForm.view "" settings model)
 
 
 viewList : Model -> Html Msg