Merge pull request #1260 from Moerfi666/recursive_mail_scan

Add a recursive mail folder scan option to ScanMailboxTask

modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

@@ -23,6 +23,8 @@ case class ScanMailboxArgs(
     account: AccountId,
     // the configured imap connection
     imapConnection: Ident,
+    // scan folders recursively
+    scanRecursively: Option[Boolean],
     // what folders to search
     folders: List[String],
     // only select mails received since then

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -24,6 +24,7 @@ import docspell.store.records._
 
 import _root_.io.circe.syntax._
 import emil.SearchQuery.{All, ReceivedDate}
+import emil.SearchResult.searchResultMonoid
 import emil.javamail.syntax._
 import emil.{MimeType => _, _}
 
@@ -98,7 +99,12 @@ object ScanMailboxTask {
       if (acc.noneLeft(name)) acc.pure[F]
       else
         mailer
-          .run(impl.handleFolder(theEmil.access, upload)(name))
+          .run(
+            impl.handleFolder(theEmil.access, upload)(
+              name,
+              ctx.args.scanRecursively.getOrElse(false)
+            )
+          )
           .map(_ ++ acc)
 
     Stream
@@ -155,12 +161,15 @@ object ScanMailboxTask {
       MailOp(_ => f(ctx.logger))
 
     def handleFolder[C](a: Access[F, C], upload: OUpload[F])(
-        name: String
+        name: String,
+        scanRecursively: Boolean
     ): MailOp[F, C, ScanResult] =
       for {
         _ <- Kleisli.liftF(ctx.logger.info(s"Processing folder $name"))
         folder <- requireFolder(a)(name)
-        search <- searchMails(a)(folder)
+        search <-
+          if (scanRecursively) searchMailsRecursively(a)(folder)
+          else searchMails(a)(folder)
         items = search.mails.map(MailHeaderItem(_))
         headers <- Kleisli.liftF(
           filterSubjects(items).flatMap(filterMessageIds)
@@ -175,6 +184,15 @@ object ScanMailboxTask {
           .map(_.toRight(new Exception(s"Folder '$name' not found")))
           .mapF(_.rethrow)
 
+    def searchMailsRecursively[C](
+        a: Access[F, C]
+    )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] =
+      for {
+        subFolders <- a.listFoldersRecursive(Some(folder))
+        foldersToSearch = Vector(folder) ++ subFolders
+        search <- foldersToSearch.traverse(searchMails(a))
+      } yield searchResultMonoid.combineAll(search)
+
     def searchMails[C](
         a: Access[F, C]
     )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = {

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -6407,6 +6407,7 @@ components:
       required:
         - id
         - enabled
+        - scanRecursively
         - imapConnection
         - schedule
         - folders
@@ -6426,6 +6427,10 @@ components:
           type: array
           items:
             type: string
+        scanRecursively:
+          type: boolean
+          description: |
+            Scan folders recursively for new mails.
         schedule:
           type: string
           format: calevent

modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala

@@ -115,6 +115,7 @@ object ScanMailboxRoutes {
         ScanMailboxArgs(
           user,
           settings.imapConnection,
+          Option(settings.scanRecursively),
           settings.folders,
           settings.receivedSinceHours.map(_.toLong).map(Duration.hours),
           settings.targetFolder,
@@ -150,6 +151,7 @@ object ScanMailboxRoutes {
       task.summary,
       conn.getOrElse(Ident.unsafe("")),
       task.args.folders,
+      task.args.scanRecursively.getOrElse(false),
       task.timer,
       task.args.receivedSince.map(_.hours.toInt),
       task.args.targetFolder,

modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm

@@ -68,6 +68,7 @@ type alias Model =
     , targetFolder : Maybe String
     , foldersModel : Comp.StringListInput.Model
     , folders : List String
+    , scanRecursively : Bool
     , direction : Maybe Direction
     , schedule : Maybe CalEvent
     , scheduleModel : Comp.CalEventInput.Model
@@ -156,6 +157,7 @@ type Msg
     | ReceivedHoursMsg Comp.IntField.Msg
     | SetTargetFolder String
     | FoldersMsg Comp.StringListInput.Msg
+    | ToggleScanRecursively
     | DirectionMsg (Maybe Direction)
     | YesNoDeleteMsg Comp.YesNoDimmer.Msg
     | GetFolderResp (Result Http.Error FolderList)
@@ -200,6 +202,7 @@ initWith flags s =
         , receivedHours = s.receivedSinceHours
         , targetFolder = s.targetFolder
         , folders = s.folders
+        , scanRecursively = s.scanRecursively
         , schedule = Just newSchedule
         , direction = Maybe.andThen Data.Direction.fromString s.direction
         , scheduleModel = sm
@@ -246,6 +249,7 @@ init flags =
       , receivedHoursModel = Comp.IntField.init (Just 1) Nothing True
       , foldersModel = Comp.StringListInput.init
       , folders = []
+      , scanRecursively = False
       , targetFolder = Nothing
       , direction = Nothing
       , schedule = Just initialSchedule
@@ -316,6 +320,7 @@ makeSettings model =
                 , deleteMail = model.deleteMail
                 , targetFolder = model.targetFolder
                 , folders = folders
+                , scanRecursively = model.scanRecursively
                 , direction = Maybe.map Data.Direction.asString model.direction
                 , schedule = Data.CalEvent.makeEvent timer
                 , itemFolder = model.itemFolderId
@@ -497,6 +502,12 @@ update flags tz msg model =
             , Cmd.none
             )
 
+        ToggleScanRecursively ->
+            ( { model | scanRecursively = not model.scanRecursively }
+            , NoAction
+            , Cmd.none
+            )
+
         DirectionMsg md ->
             ( { model | direction = md }
             , NoAction
@@ -961,6 +972,17 @@ viewGeneral2 texts settings model =
 viewProcessing2 : Texts -> Model -> List (Html Msg)
 viewProcessing2 texts model =
     [ div [ class "mb-4" ]
+        [ MB.viewItem <|
+            MB.Checkbox
+                { id = "scanmail-scan-recursively"
+                , value = model.scanRecursively
+                , label = texts.scanRecursivelyLabel
+                , tagger = \_ -> ToggleScanRecursively
+                }
+            , span [ class "opacity-50 text-sm mt-1" ]
+            [ Markdown.toHtml [] texts.scanRecursivelyInfo ]
+        ]
+    , div [ class "mb-4" ]
         [ label [ class S.inputLabel ]
             [ text texts.folders
             , B.inputRequired

modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm

@@ -68,6 +68,8 @@ type alias Texts =
     , tagsInfo : String
     , documentLanguage : String
     , documentLanguageInfo : String
+    , scanRecursivelyInfo : String
+    , scanRecursivelyLabel : String
     , schedule : String
     , scheduleClickForHelp : String
     , scheduleInfo : String
@@ -149,6 +151,8 @@ gb tz =
     , documentLanguageInfo =
         "Used for text extraction and text analysis. The "
             ++ "collective's default language is used, if not specified here."
+    , scanRecursivelyInfo = "Scan the sub-folders of the given folders for emails too."
+    , scanRecursivelyLabel = "Scan folders recursively"
     , schedule = "Schedule"
     , scheduleClickForHelp = "Click here for help"
     , scheduleInfo =
@@ -229,6 +233,8 @@ kann hier ein Wert für alle festgelegt werden. Bei 'Automatisch' wird auf den S
     , documentLanguageInfo =
         "Wird für Texterkennung und -analyse verwendet. Die Standardsprache des Kollektivs "
             ++ "wird verwendet, falls hier nicht angegeben."
+    , scanRecursivelyInfo = "Auch die Unterordner der gegebenen Ordner nach E-Mails durchsuchen."
+    , scanRecursivelyLabel = "Ordner rekursiv scannen"
     , schedule = "Zeitplan"
     , scheduleClickForHelp = "Klicke für Hilfe"
     , scheduleInfo =
@@ -311,6 +317,8 @@ fr tz =
     , documentLanguageInfo =
         "Utilisé pour l'extraction et l'analyse du texte. La langue"
             ++ "par défaut du groupe est utilisée, si non spécifié"
+    , scanRecursivelyInfo = "Rechercher également les mails dans les sous-dossiers des dossiers spécifiés."
+    , scanRecursivelyLabel = "Analyse récursive des dossiers"
     , schedule = "Programmation"
     , scheduleClickForHelp = "Cliquer pour l'aide"
     , scheduleInfo =