Merge pull request #590 from eikek/scan-mailbox-filter

Refactor scan mailbox form and add flag for post-processing
This commit is contained in:
mergify[bot] 2021-01-24 01:06:51 +00:00 committed by GitHub
commit 6cc9c159d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 419 additions and 278 deletions

View File

@ -37,7 +37,9 @@ case class ScanMailboxArgs(
// a glob filter for the mail subject
subjectFilter: Option[Glob],
// the language for extraction and analysis
language: Option[Language]
language: Option[Language],
// apply additional filter to all mails or only imported
postHandleAll: Option[Boolean]
)
object ScanMailboxArgs {

View File

@ -14,6 +14,7 @@ import docspell.joex.scheduler.{Context, Task}
import docspell.store.queries.QOrganization
import docspell.store.records._
import _root_.io.circe.syntax._
import emil.SearchQuery.{All, ReceivedDate}
import emil.javamail.syntax._
import emil.{MimeType => _, _}
@ -31,8 +32,9 @@ object ScanMailboxTask {
Task { ctx =>
for {
_ <- ctx.logger.info(
s"Start importing mails for user ${ctx.args.account.user.id}"
s"=== Start importing mails for user ${ctx.args.account.user.id}"
)
_ <- ctx.logger.debug(s"Settings: ${ctx.args.asJson.noSpaces}")
mailCfg <- getMailSettings(ctx)
folders = ctx.args.folders.mkString(", ")
userId = ctx.args.account.user
@ -249,25 +251,29 @@ object ScanMailboxTask {
.getOrElse(Direction.Incoming)
}
def postHandle[C](a: Access[F, C])(mh: MailHeader): MailOp[F, C, Unit] =
def postHandle[C](a: Access[F, C])(mh: MailHeaderItem): MailOp[F, C, Unit] = {
val postHandleAll = ctx.args.postHandleAll.exists(identity)
ctx.args.targetFolder match {
case Some(tf) =>
logOp(_.debug(s"Post handling mail: ${mh.subject} - moving to folder: $tf"))
case Some(tf) if postHandleAll || mh.process =>
logOp(
_.debug(s"Post handling mail: ${mh.mh.subject} - moving to folder: $tf")
)
.flatMap(_ =>
a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh, folder))
a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh.mh, folder))
)
case None if ctx.args.deleteMail =>
logOp(_.debug(s"Post handling mail: ${mh.subject} - deleting mail.")).flatMap(
_ =>
a.deleteMail(mh).flatMapF { r =>
case None if ctx.args.deleteMail && (postHandleAll || mh.process) =>
logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - deleting mail."))
.flatMap(_ =>
a.deleteMail(mh.mh).flatMapF { r =>
if (r.count == 0)
ctx.logger.warn(s"Mail could not be deleted!")
else ().pure[F]
}
)
case None =>
logOp(_.debug(s"Post handling mail: ${mh.subject} - no handling defined!"))
)
case _ =>
logOp(_.debug(s"Post handling mail: ${mh.mh.subject} - no handling defined!"))
}
}
def submitMail(upload: OUpload[F], args: Args)(
mail: Mail[F]
@ -321,7 +327,7 @@ object ScanMailboxTask {
Kleisli.liftF(
ctx.logger.warn(s"Error submitting '${mh.mh.subject}': ${ex.getMessage}")
),
_ => postHandle(a)(mh.mh)
_ => postHandle(a)(mh)
)
} yield ()
}

View File

@ -3904,6 +3904,8 @@ components:
processing mails.
type: string
format: language
postHandleAll:
type: boolean
ImapSettingsList:
description: |

View File

@ -117,7 +117,8 @@ object ScanMailboxRoutes {
settings.fileFilter,
settings.tags.map(_.items),
settings.subjectFilter,
settings.language
settings.language,
settings.postHandleAll
)
)
)
@ -149,6 +150,7 @@ object ScanMailboxRoutes {
task.args.tags.map(StringList.apply),
task.args.fileFilter,
task.args.subjectFilter,
task.args.language
task.args.language,
task.args.postHandleAll
)
}

View File

@ -68,6 +68,8 @@ type alias Model =
, subjectFilter : Maybe String
, languageModel : Comp.FixedDropdown.Model Language
, language : Maybe Language
, postHandleAll : Bool
, menuTab : MenuTab
}
@ -79,6 +81,15 @@ type Action
| NoAction
type MenuTab
= TabGeneral
| TabProcessing
| TabAdditionalFilter
| TabPostProcessing
| TabMetadata
| TabSchedule
type Msg
= Submit
| Cancel
@ -102,6 +113,8 @@ type Msg
| SetSubjectFilter String
| LanguageMsg (Comp.FixedDropdown.Msg Language)
| RemoveLanguage
| TogglePostHandleAll
| SetMenuTab MenuTab
initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg )
@ -147,6 +160,8 @@ initWith flags s =
, languageModel =
Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
, language = Maybe.andThen Data.Language.fromString s.language
, postHandleAll = Maybe.withDefault False s.postHandleAll
, menuTab = TabGeneral
}
, Cmd.batch
[ Api.getImapSettings flags "" ConnResp
@ -200,6 +215,8 @@ init flags =
, languageModel =
Comp.FixedDropdown.init (List.map mkLanguageItem Data.Language.all)
, language = Nothing
, postHandleAll = False
, menuTab = TabGeneral
}
, Cmd.batch
[ Api.getImapSettings flags "" ConnResp
@ -232,7 +249,7 @@ makeSettings model =
infolders =
if model.folders == [] then
Invalid [ "No folders given" ] []
Invalid [ "No processing folders given" ] []
else
Valid model.folders
@ -260,6 +277,7 @@ makeSettings model =
|> StringList
|> Just
, language = Maybe.map Data.Language.toIso3 model.language
, postHandleAll = Just model.postHandleAll
}
in
Data.Validated.map3 make
@ -646,6 +664,18 @@ update flags msg model =
, Cmd.none
)
TogglePostHandleAll ->
( { model | postHandleAll = not model.postHandleAll }
, NoAction
, Cmd.none
)
SetMenuTab tab ->
( { model | menuTab = tab }
, NoAction
, Cmd.none
)
--- View
@ -674,267 +704,27 @@ view extraClasses settings model =
, ( "success", isFormSuccess model )
]
]
[ Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
, div
[ classList
[ ( "ui dimmer", True )
, ( "active", model.loading > 0 )
]
]
[ div [ class "ui text loader" ]
[ text "Loading..."
]
]
, div [ class "inline field" ]
[ div [ class "ui checkbox" ]
[ input
[ type_ "checkbox"
, onCheck (\_ -> ToggleEnabled)
, checked model.enabled
]
[]
, label [] [ text "Enabled" ]
]
, span [ class "small-info" ]
[ text "Enable or disable this task."
]
]
, div [ class "required field" ]
[ label [] [ text "Mailbox" ]
, Html.map ConnMsg (Comp.Dropdown.view settings model.connectionModel)
, span [ class "small-info" ]
[ text "The IMAP connection to use when sending notification mails."
]
]
, div [ class "required field" ]
[ label [] [ text "Folders" ]
, Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel)
, span [ class "small-info" ]
[ text "The folders to go through"
]
]
, div [ class "field" ]
[ label [] [ text "Target folder" ]
, input
[ type_ "text"
, onInput SetTargetFolder
, Maybe.withDefault "" model.targetFolder |> value
]
[]
, span [ class "small-info" ]
[ text "Move all mails successfully submitted into this folder."
]
]
, div [ class "inline field" ]
[ div [ class "ui checkbox" ]
[ input
[ type_ "checkbox"
, onCheck (\_ -> ToggleDeleteMail)
, checked model.deleteMail
]
[]
, label [] [ text "Delete imported mails" ]
]
, span [ class "small-info" ]
[ text "Whether to delete all mails successfully imported into docspell. This only applies if "
, em [] [ text "target folder" ]
, text " is not set."
]
]
, div [ class "ui dividing header" ]
[ text "Filter"
]
, Html.map ReceivedHoursMsg
(Comp.IntField.viewWithInfo
"Select mails newer than `now - receivedHours`"
model.receivedHours
"field"
model.receivedHoursModel
[ viewMenu model
, div [ class "ui bottom attached segment" ]
(case model.menuTab of
TabGeneral ->
viewGeneral settings model
TabProcessing ->
viewProcessing model
TabAdditionalFilter ->
viewAdditionalFilter model
TabPostProcessing ->
viewPostProcessing model
TabMetadata ->
viewMetadata settings model
TabSchedule ->
viewSchedule model
)
, div
[ class "field"
]
[ label [] [ text "File Filter" ]
, input
[ type_ "text"
, onInput SetFileFilter
, placeholder "File Filter"
, model.fileFilter
|> Maybe.withDefault ""
|> value
]
[]
, div [ class "small-info" ]
[ text "Specify a file glob to filter attachments. For example, to only extract pdf files: "
, code []
[ text "*.pdf"
]
, text ". If you want to include the mail body, allow html files or "
, code []
[ text "mail.html"
]
, text ". Globs can be combined via OR, like this: "
, code []
[ text "*.pdf|mail.html"
]
, text ". No file filter defaults to "
, code []
[ text "*"
]
, text " that includes all"
]
]
, div
[ class "field"
]
[ label [] [ text "Subject Filter" ]
, input
[ type_ "text"
, onInput SetSubjectFilter
, placeholder "Subject Filter"
, model.subjectFilter
|> Maybe.withDefault ""
|> value
]
[]
, div [ class "small-info" ]
[ text "Specify a file glob to filter mails by subject. For example: "
, code []
[ text "*Scanned Document*"
]
, text ". No file filter defaults to "
, code []
[ text "*"
]
, text " that includes all"
]
]
, div [ class "ui dividing header" ]
[ text "Metadata"
]
, div [ class "required field" ]
[ label [] [ text "Item direction" ]
, div [ class "grouped fields" ]
[ div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Nothing)
, onCheck (\_ -> DirectionMsg Nothing)
]
[]
, label [] [ text "Automatic" ]
]
]
, div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Just Incoming)
, onCheck (\_ -> DirectionMsg (Just Incoming))
]
[]
, label [] [ text "Incoming" ]
]
]
, div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Just Outgoing)
, onCheck (\_ -> DirectionMsg (Just Outgoing))
]
[]
, label [] [ text "Outgoing" ]
]
]
, span [ class "small-info" ]
[ text "Sets the direction for an item. If you know all mails are incoming or "
, text "outgoing, you can set it here. Otherwise it will be guessed from looking "
, text "at sender and receiver."
]
]
]
, div [ class "field" ]
[ label []
[ text "Item Folder"
]
, Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel)
, span [ class "small-info" ]
[ text "Put all items from this mailbox into the selected folder"
]
, div
[ classList
[ ( "ui warning message", True )
, ( "hidden", isFolderMember model )
]
]
[ Markdown.toHtml [] """
You are **not a member** of this folder. Items created from mails in
this mailbox will be **hidden** from any search results. Use a folder
where you are a member of to make items visible. This message will
disappear then.
"""
]
]
, div [ class "field" ]
[ label [] [ text "Tags" ]
, Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
, div [ class "small-info" ]
[ text "Choose tags that should be applied to items."
]
]
, div [ class "field" ]
[ label []
[ text "Language"
]
, div [ class "ui action input" ]
[ Html.map LanguageMsg
(Comp.FixedDropdown.viewStyled "fluid"
(Maybe.map mkLanguageItem model.language)
model.languageModel
)
, a
[ class "ui icon button"
, href "#"
, onClick RemoveLanguage
]
[ i [ class "delete icon" ] []
]
]
, div [ class "small-info" ]
[ text "Used for text extraction and text analysis. The "
, text "collective's default language is used, if not specified here."
]
]
, div [ class "ui dividing header" ]
[ text "Schedule"
]
, div [ class "required field" ]
[ label []
[ text "Schedule"
, a
[ class "right-float"
, href "https://github.com/eikek/calev#what-are-calendar-events"
, target "_blank"
]
[ i [ class "help icon" ] []
, text "Click here for help"
]
]
, Html.map CalEventMsg
(Comp.CalEventInput.view ""
(Data.Validated.value model.schedule)
model.scheduleModel
)
, span [ class "small-info" ]
[ text "Specify how often and when this task should run. "
, text "Use English 3-letter weekdays. Either a single value, "
, text "a list (ex. 1,2,3), a range (ex. 1..3) or a '*' (meaning all) "
, text "is allowed for each part."
]
]
, div [ class "ui divider" ] []
, div
[ classList
[ ( "ui message", True )
@ -974,9 +764,348 @@ disappear then.
]
[ text "Start Once"
]
, Html.map YesNoDeleteMsg (Comp.YesNoDimmer.view model.yesNoDelete)
, div
[ classList
[ ( "ui dimmer", True )
, ( "active", model.loading > 0 )
, ( "invisible hidden", model.loading == 0 )
]
]
[ div [ class "ui text loader" ]
[ text "Loading..."
]
]
]
viewMenu : Model -> Html Msg
viewMenu model =
let
tabLink tab txt =
a
[ class "item"
, classList
[ ( "active", model.menuTab == tab ) ]
, href "#"
, onClick (SetMenuTab tab)
]
[ text txt
]
in
div [ class "ui top attached stacked tabular menu" ]
[ tabLink TabGeneral "General"
, tabLink TabProcessing "Processing"
, tabLink TabAdditionalFilter "Additional Filter"
, tabLink TabPostProcessing "Post Processing"
, tabLink TabMetadata "Metadata"
, tabLink TabSchedule "Schedule"
]
viewGeneral : UiSettings -> Model -> List (Html Msg)
viewGeneral settings model =
[ div [ class "inline field" ]
[ div [ class "ui checkbox" ]
[ input
[ type_ "checkbox"
, onCheck (\_ -> ToggleEnabled)
, checked model.enabled
]
[]
, label [] [ text "Enabled" ]
]
, span [ class "small-info" ]
[ text "Enable or disable this task."
]
]
, div [ class "required field" ]
[ label [] [ text "Mailbox" ]
, Html.map ConnMsg (Comp.Dropdown.view settings model.connectionModel)
, span [ class "small-info" ]
[ text "The IMAP connection to use when sending notification mails."
]
]
]
viewProcessing : Model -> List (Html Msg)
viewProcessing model =
[ div [ class "ui message" ]
[ text "These settings define which mails are fetched from the mail server."
]
, div [ class "required field" ]
[ label [] [ text "Folders" ]
, Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel)
, span [ class "small-info" ]
[ text "The folders to go through"
]
]
, Html.map ReceivedHoursMsg
(Comp.IntField.viewWithInfo
"Select mails newer than `now - receivedHours`"
model.receivedHours
"field"
model.receivedHoursModel
)
]
viewAdditionalFilter : Model -> List (Html Msg)
viewAdditionalFilter model =
[ div [ class "ui message" ]
[ text "These filters are applied to mails that have been fetched from the "
, text "mailbox to select those that should be imported."
]
, div
[ class "field"
]
[ label [] [ text "File Filter" ]
, input
[ type_ "text"
, onInput SetFileFilter
, placeholder "File Filter"
, model.fileFilter
|> Maybe.withDefault ""
|> value
]
[]
, div [ class "small-info" ]
[ text "Specify a file glob to filter attachments. For example, to only extract pdf files: "
, code []
[ text "*.pdf"
]
, text ". If you want to include the mail body, allow html files or "
, code []
[ text "mail.html"
]
, text ". Globs can be combined via OR, like this: "
, code []
[ text "*.pdf|mail.html"
]
, text ". No file filter defaults to "
, code []
[ text "*"
]
, text " that includes all"
]
]
, div
[ class "field"
]
[ label [] [ text "Subject Filter" ]
, input
[ type_ "text"
, onInput SetSubjectFilter
, placeholder "Subject Filter"
, model.subjectFilter
|> Maybe.withDefault ""
|> value
]
[]
, div [ class "small-info" ]
[ text "Specify a file glob to filter mails by subject. For example: "
, code []
[ text "*Scanned Document*"
]
, text ". No file filter defaults to "
, code []
[ text "*"
]
, text " that includes all"
]
]
]
viewPostProcessing : Model -> List (Html Msg)
viewPostProcessing model =
[ div [ class "ui message" ]
[ text "This defines what happens to mails that have been downloaded."
]
, div [ class "inline field" ]
[ div [ class "ui checkbox" ]
[ input
[ type_ "checkbox"
, onCheck (\_ -> TogglePostHandleAll)
, checked model.postHandleAll
]
[]
, label [] [ text "Apply post-processing to all fetched mails." ]
]
, span [ class "small-info" ]
[ text "When mails are not imported due to the 'Additional Filters', this flag can "
, text "control whether they should be moved to a target folder or deleted (whatever is "
, text "defined here) nevertheless. If unchecked only imported mails "
, text "are post-processed, others stay where they are."
]
]
, div [ class "field" ]
[ label [] [ text "Target folder" ]
, input
[ type_ "text"
, onInput SetTargetFolder
, Maybe.withDefault "" model.targetFolder |> value
]
[]
, span [ class "small-info" ]
[ text "Move mails into this folder."
]
]
, div [ class "inline field" ]
[ div [ class "ui checkbox" ]
[ input
[ type_ "checkbox"
, onCheck (\_ -> ToggleDeleteMail)
, checked model.deleteMail
]
[]
, label [] [ text "Delete imported mails" ]
]
, span [ class "small-info" ]
[ text "Whether to delete all mails fetched by docspell. This only applies if "
, em [] [ text "target folder" ]
, text " is not set."
]
]
]
viewMetadata : UiSettings -> Model -> List (Html Msg)
viewMetadata settings model =
[ div [ class "ui message" ]
[ text "Define metadata that should be attached to all items created by this task."
]
, div [ class "required field" ]
[ label [] [ text "Item direction" ]
, div [ class "grouped fields" ]
[ div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Nothing)
, onCheck (\_ -> DirectionMsg Nothing)
]
[]
, label [] [ text "Automatic" ]
]
]
, div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Just Incoming)
, onCheck (\_ -> DirectionMsg (Just Incoming))
]
[]
, label [] [ text "Incoming" ]
]
]
, div [ class "field" ]
[ div [ class "ui radio checkbox" ]
[ input
[ type_ "radio"
, checked (model.direction == Just Outgoing)
, onCheck (\_ -> DirectionMsg (Just Outgoing))
]
[]
, label [] [ text "Outgoing" ]
]
]
, span [ class "small-info" ]
[ text "Sets the direction for an item. If you know all mails are incoming or "
, text "outgoing, you can set it here. Otherwise it will be guessed from looking "
, text "at sender and receiver."
]
]
]
, div [ class "field" ]
[ label []
[ text "Item Folder"
]
, Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel)
, span [ class "small-info" ]
[ text "Put all items from this mailbox into the selected folder"
]
, div
[ classList
[ ( "ui warning message", True )
, ( "hidden", isFolderMember model )
]
]
[ Markdown.toHtml [] """
You are **not a member** of this folder. Items created from mails in
this mailbox will be **hidden** from any search results. Use a folder
where you are a member of to make items visible. This message will
disappear then.
"""
]
]
, div [ class "field" ]
[ label [] [ text "Tags" ]
, Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
, div [ class "small-info" ]
[ text "Choose tags that should be applied to items."
]
]
, div [ class "field" ]
[ label []
[ text "Language"
]
, div [ class "ui action input" ]
[ Html.map LanguageMsg
(Comp.FixedDropdown.viewStyled "fluid"
(Maybe.map mkLanguageItem model.language)
model.languageModel
)
, a
[ class "ui icon button"
, href "#"
, onClick RemoveLanguage
]
[ i [ class "delete icon" ] []
]
]
, div [ class "small-info" ]
[ text "Used for text extraction and text analysis. The "
, text "collective's default language is used, if not specified here."
]
]
]
viewSchedule : Model -> List (Html Msg)
viewSchedule model =
[ div [ class "ui message" ]
[ text "Define when mails should be imported."
]
, div [ class "required field" ]
[ label []
[ text "Schedule"
, a
[ class "right-float"
, href "https://github.com/eikek/calev#what-are-calendar-events"
, target "_blank"
]
[ i [ class "help icon" ] []
, text "Click here for help"
]
]
, Html.map CalEventMsg
(Comp.CalEventInput.view ""
(Data.Validated.value model.schedule)
model.scheduleModel
)
, span [ class "small-info" ]
[ text "Specify how often and when this task should run. "
, text "Use English 3-letter weekdays. Either a single value, "
, text "a list (ex. 1,2,3), a range (ex. 1..3) or a '*' (meaning all) "
, text "is allowed for each part."
]
]
]
isFolderMember : Model -> Bool
isFolderMember model =
let

View File

@ -247,7 +247,7 @@ view settings model =
viewForm : UiSettings -> Comp.ScanMailboxForm.Model -> Html Msg
viewForm settings model =
Html.map DetailMsg (Comp.ScanMailboxForm.view "segment" settings model)
Html.map DetailMsg (Comp.ScanMailboxForm.view "" settings model)
viewList : Model -> Html Msg