From 39f2f8cc1b7b5f41fd2aa6b50269b92872677bb9 Mon Sep 17 00:00:00 2001 From: moerfi Date: Fri, 7 Jan 2022 22:18:47 +0100 Subject: [PATCH 1/5] Add a recursive mail folder scan option to ScanMailboxTask --- .../docspell/common/ScanMailboxArgs.scala | 2 ++ .../joex/scanmailbox/ScanMailboxTask.scala | 22 ++++++++++++++++--- .../src/main/resources/docspell-openapi.yml | 5 +++++ .../restserver/routes/ScanMailboxRoutes.scala | 2 ++ .../src/main/elm/Comp/ScanMailboxForm.elm | 22 +++++++++++++++++++ .../elm/Messages/Comp/ScanMailboxForm.elm | 6 +++++ 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index f23ccad3..2a025386 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -23,6 +23,8 @@ case class ScanMailboxArgs( account: AccountId, // the configured imap connection imapConnection: Ident, + // scan folders recursively + scanRecursively: Boolean, // what folders to search folders: List[String], // only select mails received since then diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 039dee08..acb8f7c7 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -98,7 +98,7 @@ object ScanMailboxTask { if (acc.noneLeft(name)) acc.pure[F] else mailer - .run(impl.handleFolder(theEmil.access, upload)(name)) + .run(impl.handleFolder(theEmil.access, upload)(name, ctx.args.scanRecursively)) .map(_ ++ acc) Stream @@ -155,12 +155,15 @@ object ScanMailboxTask { MailOp(_ => f(ctx.logger)) def handleFolder[C](a: Access[F, C], upload: OUpload[F])( - name: String + name: String, + scanRecursively: Boolean ): MailOp[F, C, ScanResult] = for { _ <- Kleisli.liftF(ctx.logger.info(s"Processing folder $name")) folder <- requireFolder(a)(name) - search <- searchMails(a)(folder) + search <- + if (scanRecursively) searchMailsRecursively(a)(folder) + else searchMails(a)(folder) items = search.mails.map(MailHeaderItem(_)) headers <- Kleisli.liftF( filterSubjects(items).flatMap(filterMessageIds) @@ -175,6 +178,19 @@ object ScanMailboxTask { .map(_.toRight(new Exception(s"Folder '$name' not found"))) .mapF(_.rethrow) + def searchMailsRecursively[C]( + a: Access[F, C] + )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = + for { + subFolders <- a.listFoldersRecursive(Some(folder)) + search <- subFolders.foldLeft(searchMails(a)(folder)) { (result, folder) => + for { + res <- result + search <- searchMails(a)(folder) + } yield SearchResult(res.mails ++ search.mails, res.count + search.count) + } + } yield search + def searchMails[C]( a: Access[F, C] )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = { diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 669825e4..ed349c57 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -6407,6 +6407,7 @@ components: required: - id - enabled + - scanRecursively - imapConnection - schedule - folders @@ -6426,6 +6427,10 @@ components: type: array items: type: string + scanRecursively: + type: boolean + description: | + Scan folders recursively for new mails. schedule: type: string format: calevent diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index df271180..87b0a9e6 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -115,6 +115,7 @@ object ScanMailboxRoutes { ScanMailboxArgs( user, settings.imapConnection, + settings.scanRecursively, settings.folders, settings.receivedSinceHours.map(_.toLong).map(Duration.hours), settings.targetFolder, @@ -150,6 +151,7 @@ object ScanMailboxRoutes { task.summary, conn.getOrElse(Ident.unsafe("")), task.args.folders, + task.args.scanRecursively, task.timer, task.args.receivedSince.map(_.hours.toInt), task.args.targetFolder, diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm index 2bd9ff2d..7516326f 100644 --- a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm @@ -68,6 +68,7 @@ type alias Model = , targetFolder : Maybe String , foldersModel : Comp.StringListInput.Model , folders : List String + , scanRecursively : Bool , direction : Maybe Direction , schedule : Maybe CalEvent , scheduleModel : Comp.CalEventInput.Model @@ -156,6 +157,7 @@ type Msg | ReceivedHoursMsg Comp.IntField.Msg | SetTargetFolder String | FoldersMsg Comp.StringListInput.Msg + | ToggleScanRecursively | DirectionMsg (Maybe Direction) | YesNoDeleteMsg Comp.YesNoDimmer.Msg | GetFolderResp (Result Http.Error FolderList) @@ -200,6 +202,7 @@ initWith flags s = , receivedHours = s.receivedSinceHours , targetFolder = s.targetFolder , folders = s.folders + , scanRecursively = s.scanRecursively , schedule = Just newSchedule , direction = Maybe.andThen Data.Direction.fromString s.direction , scheduleModel = sm @@ -246,6 +249,7 @@ init flags = , receivedHoursModel = Comp.IntField.init (Just 1) Nothing True , foldersModel = Comp.StringListInput.init , folders = [] + , scanRecursively = False , targetFolder = Nothing , direction = Nothing , schedule = Just initialSchedule @@ -316,6 +320,7 @@ makeSettings model = , deleteMail = model.deleteMail , targetFolder = model.targetFolder , folders = folders + , scanRecursively = model.scanRecursively , direction = Maybe.map Data.Direction.asString model.direction , schedule = Data.CalEvent.makeEvent timer , itemFolder = model.itemFolderId @@ -497,6 +502,12 @@ update flags tz msg model = , Cmd.none ) + ToggleScanRecursively -> + ( { model | scanRecursively = not model.scanRecursively } + , NoAction + , Cmd.none + ) + DirectionMsg md -> ( { model | direction = md } , NoAction @@ -961,6 +972,17 @@ viewGeneral2 texts settings model = viewProcessing2 : Texts -> Model -> List (Html Msg) viewProcessing2 texts model = [ div [ class "mb-4" ] + [ MB.viewItem <| + MB.Checkbox + { id = "scanmail-scan-recursively" + , value = model.scanRecursively + , label = texts.scanRecursivelyLabel + , tagger = \_ -> ToggleScanRecursively + } + , span [ class "opacity-50 text-sm mt-1" ] + [ Markdown.toHtml [] texts.scanRecursivelyInfo ] + ] + , div [ class "mb-4" ] [ label [ class S.inputLabel ] [ text texts.folders , B.inputRequired diff --git a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm index 9f8ca1b8..c38a6536 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm @@ -68,6 +68,8 @@ type alias Texts = , tagsInfo : String , documentLanguage : String , documentLanguageInfo : String + , scanRecursivelyInfo : String + , scanRecursivelyLabel : String , schedule : String , scheduleClickForHelp : String , scheduleInfo : String @@ -149,6 +151,8 @@ gb tz = , documentLanguageInfo = "Used for text extraction and text analysis. The " ++ "collective's default language is used, if not specified here." + , scanRecursivelyInfo = "Scan the sub-folders of the given folders for emails too." + , scanRecursivelyLabel = "Scan folders recursively" , schedule = "Schedule" , scheduleClickForHelp = "Click here for help" , scheduleInfo = @@ -229,6 +233,8 @@ kann hier ein Wert für alle festgelegt werden. Bei 'Automatisch' wird auf den S , documentLanguageInfo = "Wird für Texterkennung und -analyse verwendet. Die Standardsprache des Kollektivs " ++ "wird verwendet, falls hier nicht angegeben." + , scanRecursivelyInfo = "Auch die Unterordner der gegebenen Ordner nach E-Mails durchsuchen." + , scanRecursivelyLabel = "Ordner rekursiv scannen" , schedule = "Zeitplan" , scheduleClickForHelp = "Klicke für Hilfe" , scheduleInfo = From a6e79a32a8e18e7c8fa6f74552d1347ae009eb04 Mon Sep 17 00:00:00 2001 From: moerfi Date: Sat, 22 Jan 2022 22:14:21 +0100 Subject: [PATCH 2/5] Change scanRecursively to an Option field --- .../src/main/scala/docspell/common/ScanMailboxArgs.scala | 2 +- .../scala/docspell/joex/scanmailbox/ScanMailboxTask.scala | 2 +- .../scala/docspell/restserver/routes/ScanMailboxRoutes.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala index 2a025386..9b886c90 100644 --- a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala @@ -24,7 +24,7 @@ case class ScanMailboxArgs( // the configured imap connection imapConnection: Ident, // scan folders recursively - scanRecursively: Boolean, + scanRecursively: Option[Boolean], // what folders to search folders: List[String], // only select mails received since then diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index acb8f7c7..c5cd8ff5 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -98,7 +98,7 @@ object ScanMailboxTask { if (acc.noneLeft(name)) acc.pure[F] else mailer - .run(impl.handleFolder(theEmil.access, upload)(name, ctx.args.scanRecursively)) + .run(impl.handleFolder(theEmil.access, upload)(name, ctx.args.scanRecursively.getOrElse(false))) .map(_ ++ acc) Stream diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 87b0a9e6..61f27c11 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -115,7 +115,7 @@ object ScanMailboxRoutes { ScanMailboxArgs( user, settings.imapConnection, - settings.scanRecursively, + Option(settings.scanRecursively), settings.folders, settings.receivedSinceHours.map(_.toLong).map(Duration.hours), settings.targetFolder, @@ -151,7 +151,7 @@ object ScanMailboxRoutes { task.summary, conn.getOrElse(Ident.unsafe("")), task.args.folders, - task.args.scanRecursively, + task.args.scanRecursively.getOrElse(false), task.timer, task.args.receivedSince.map(_.hours.toInt), task.args.targetFolder, From 805f57def3babacc680ff74ed0945b2b77e7205d Mon Sep 17 00:00:00 2001 From: moerfi Date: Sun, 23 Jan 2022 19:22:27 +0100 Subject: [PATCH 3/5] Simplify recursive folder scan with --- .../joex/scanmailbox/ScanMailboxTask.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index c5cd8ff5..fcd2e33f 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -24,6 +24,7 @@ import docspell.store.records._ import _root_.io.circe.syntax._ import emil.SearchQuery.{All, ReceivedDate} +import emil.SearchResult.searchResultMonoid import emil.javamail.syntax._ import emil.{MimeType => _, _} @@ -98,7 +99,12 @@ object ScanMailboxTask { if (acc.noneLeft(name)) acc.pure[F] else mailer - .run(impl.handleFolder(theEmil.access, upload)(name, ctx.args.scanRecursively.getOrElse(false))) + .run( + impl.handleFolder(theEmil.access, upload)( + name, + ctx.args.scanRecursively.getOrElse(false) + ) + ) .map(_ ++ acc) Stream @@ -183,13 +189,9 @@ object ScanMailboxTask { )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = for { subFolders <- a.listFoldersRecursive(Some(folder)) - search <- subFolders.foldLeft(searchMails(a)(folder)) { (result, folder) => - for { - res <- result - search <- searchMails(a)(folder) - } yield SearchResult(res.mails ++ search.mails, res.count + search.count) - } - } yield search + foldersToSearch = Vector(folder) ++ subFolders + search <- foldersToSearch.traverse(searchMails(a)) + } yield searchResultMonoid.combineAll(search) def searchMails[C]( a: Access[F, C] From cdbc73bba293f7808c0fbaa689362bd123a7a6c9 Mon Sep 17 00:00:00 2001 From: moerfi Date: Tue, 15 Mar 2022 22:04:31 +0100 Subject: [PATCH 4/5] Add french translations with help of a translator --- modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm index c38a6536..9bd53d37 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm @@ -317,6 +317,8 @@ fr tz = , documentLanguageInfo = "Utilisé pour l'extraction et l'analyse du texte. La langue" ++ "par défaut du groupe est utilisée, si non spécifié" + , scanRecursivelyInfo = "Recherchez également les courriels dans les sous-dossiers des dossiers donnés." + , scanRecursivelyLabel = "Analyse récursive des dossiers" , schedule = "Programmation" , scheduleClickForHelp = "Cliquer pour l'aide" , scheduleInfo = From c7da3dcfeee3f2d544d1bedd31f1cbb2bf8b0374 Mon Sep 17 00:00:00 2001 From: eikek Date: Wed, 16 Mar 2022 23:25:06 +0100 Subject: [PATCH 5/5] Fix French translation Thanks @jgiradet --- modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm index 9bd53d37..b4d42007 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/ScanMailboxForm.elm @@ -317,7 +317,7 @@ fr tz = , documentLanguageInfo = "Utilisé pour l'extraction et l'analyse du texte. La langue" ++ "par défaut du groupe est utilisée, si non spécifié" - , scanRecursivelyInfo = "Recherchez également les courriels dans les sous-dossiers des dossiers donnés." + , scanRecursivelyInfo = "Rechercher également les mails dans les sous-dossiers des dossiers spécifiés." , scanRecursivelyLabel = "Analyse récursive des dossiers" , schedule = "Programmation" , scheduleClickForHelp = "Cliquer pour l'aide"