From 39f2f8cc1b7b5f41fd2aa6b50269b92872677bb9 Mon Sep 17 00:00:00 2001 From: moerfi Date: Fri, 7 Jan 2022 22:18:47 +0100 Subject: [PATCH] Add a recursive mail folder scan option to ScanMailboxTask --- .../docspell/common/ScanMailboxArgs.scala | 2 ++ .../joex/scanmailbox/ScanMailboxTask.scala | 22 ++++++++++++++++--- .../src/main/resources/docspell-openapi.yml | 5 +++++ .../restserver/routes/ScanMailboxRoutes.scala | 2 ++ .../src/main/elm/Comp/ScanMailboxForm.elm | 22 +++++++++++++++++++ .../elm/Messages/Comp/ScanMailboxForm.elm | 6 +++++ 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index f23ccad3..2a025386 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -23,6 +23,8 @@ case class ScanMailboxArgs( account: AccountId, // the configured imap connection imapConnection: Ident, + // scan folders recursively + scanRecursively: Boolean, // what folders to search folders: List[String], // only select mails received since then diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 039dee08..acb8f7c7 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -98,7 +98,7 @@ object ScanMailboxTask { if (acc.noneLeft(name)) acc.pure[F] else mailer - .run(impl.handleFolder(theEmil.access, upload)(name)) + .run(impl.handleFolder(theEmil.access, upload)(name, ctx.args.scanRecursively)) .map(_ ++ acc) Stream @@ -155,12 +155,15 @@ object ScanMailboxTask { MailOp(_ => f(ctx.logger)) def handleFolder[C](a: Access[F, C], upload: OUpload[F])( - name: String + name: String, + scanRecursively: Boolean ): MailOp[F, C, ScanResult] = for { _ <- Kleisli.liftF(ctx.logger.info(s"Processing folder $name")) folder <- requireFolder(a)(name) - search <- searchMails(a)(folder) + search <- + if (scanRecursively) searchMailsRecursively(a)(folder) + else searchMails(a)(folder) items = search.mails.map(MailHeaderItem(_)) headers <- Kleisli.liftF( filterSubjects(items).flatMap(filterMessageIds) @@ -175,6 +178,19 @@ object ScanMailboxTask { .map(_.toRight(new Exception(s"Folder '$name' not found"))) .mapF(_.rethrow) + def searchMailsRecursively[C]( + a: Access[F, C] + )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = + for { + subFolders <- a.listFoldersRecursive(Some(folder)) + search <- subFolders.foldLeft(searchMails(a)(folder)) { (result, folder) => + for { + res <- result + search <- searchMails(a)(folder) + } yield SearchResult(res.mails ++ search.mails, res.count + search.count) + } + } yield search + def searchMails[C]( a: Access[F, C] )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = { diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 669825e4..ed349c57 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -6407,6 +6407,7 @@ components: required: - id - enabled + - scanRecursively - imapConnection - schedule - folders @@ -6426,6 +6427,10 @@ components: type: array items: type: string + scanRecursively: + type: boolean + description: | + Scan folders recursively for new mails. schedule: type: string format: calevent diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index df271180..87b0a9e6 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -115,6 +115,7 @@ object ScanMailboxRoutes { ScanMailboxArgs( user, settings.imapConnection, + settings.scanRecursively, settings.folders, settings.receivedSinceHours.map(_.toLong).map(Duration.hours), settings.targetFolder, @@ -150,6 +151,7 @@ object ScanMailboxRoutes { task.summary, conn.getOrElse(Ident.unsafe("")), task.args.folders, + task.args.scanRecursively, task.timer, task.args.receivedSince.map(_.hours.toInt), task.args.targetFolder, diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm index 2bd9ff2d..7516326f 100644 --- a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm @@ -68,6 +68,7 @@ type alias Model = , targetFolder : Maybe String , foldersModel : Comp.StringListInput.Model , folders : List String + , scanRecursively : Bool , direction : Maybe Direction , schedule : Maybe CalEvent , scheduleModel : Comp.CalEventInput.Model @@ -156,6 +157,7 @@ type Msg | ReceivedHoursMsg Comp.IntField.Msg | SetTargetFolder String | FoldersMsg Comp.StringListInput.Msg + | ToggleScanRecursively | DirectionMsg (Maybe Direction) | YesNoDeleteMsg Comp.YesNoDimmer.Msg | GetFolderResp (Result Http.Error FolderList) @@ -200,6 +202,7 @@ initWith flags s = , receivedHours = s.receivedSinceHours , targetFolder = s.targetFolder , folders = s.folders + , scanRecursively = s.scanRecursively , schedule = Just newSchedule , direction = Maybe.andThen Data.Direction.fromString s.direction , scheduleModel = sm @@ -246,6 +249,7 @@ init flags = , receivedHoursModel = Comp.IntField.init (Just 1) Nothing True , foldersModel = Comp.StringListInput.init , folders = [] + , scanRecursively = False , targetFolder = Nothing , direction = Nothing , schedule = Just initialSchedule @@ -316,6 +320,7 @@ makeSettings model = , deleteMail = model.deleteMail , targetFolder = model.targetFolder , folders = folders + , scanRecursively = model.scanRecursively , direction = Maybe.map Data.Direction.asString model.direction , schedule = Data.CalEvent.makeEvent timer , itemFolder = model.itemFolderId @@ -497,6 +502,12 @@ update flags tz msg model = , Cmd.none ) + ToggleScanRecursively -> + ( { model | scanRecursively = not model.scanRecursively } + , NoAction + , Cmd.none + ) + DirectionMsg md -> ( { model | direction = md } , NoAction @@ -961,6 +972,17 @@ viewGeneral2 texts settings model = viewProcessing2 : Texts -> Model -> List (Html Msg) viewProcessing2 texts model = [ div [ class "mb-4" ] + [ MB.viewItem <| + MB.Checkbox + { id = "scanmail-scan-recursively" + , value = model.scanRecursively + , label = texts.scanRecursivelyLabel + , tagger = \_ -> ToggleScanRecursively + } + , span [ class "opacity-50 text-sm mt-1" ] + [ Markdown.toHtml [] texts.scanRecursivelyInfo ] + ] + , div [ class "mb-4" ] [ label [ class S.inputLabel ] [ text texts.folders , B.inputRequired diff --git a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm index 9f8ca1b8..c38a6536 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm @@ -68,6 +68,8 @@ type alias Texts = , tagsInfo : String , documentLanguage : String , documentLanguageInfo : String + , scanRecursivelyInfo : String + , scanRecursivelyLabel : String , schedule : String , scheduleClickForHelp : String , scheduleInfo : String @@ -149,6 +151,8 @@ gb tz = , documentLanguageInfo = "Used for text extraction and text analysis. The " ++ "collective's default language is used, if not specified here." + , scanRecursivelyInfo = "Scan the sub-folders of the given folders for emails too." + , scanRecursivelyLabel = "Scan folders recursively" , schedule = "Schedule" , scheduleClickForHelp = "Click here for help" , scheduleInfo = @@ -229,6 +233,8 @@ kann hier ein Wert für alle festgelegt werden. Bei 'Automatisch' wird auf den S , documentLanguageInfo = "Wird für Texterkennung und -analyse verwendet. Die Standardsprache des Kollektivs " ++ "wird verwendet, falls hier nicht angegeben." + , scanRecursivelyInfo = "Auch die Unterordner der gegebenen Ordner nach E-Mails durchsuchen." + , scanRecursivelyLabel = "Ordner rekursiv scannen" , schedule = "Zeitplan" , scheduleClickForHelp = "Klicke für Hilfe" , scheduleInfo =