From 058c31e1f629b47469aa61ee46f87ffc50c6fb96 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 11 Mar 2021 21:43:06 +0100 Subject: [PATCH 1/4] Reprocessing now sets metadata to an item if not in state confirmed When reprocessing an item, the metadat of all *files* are replaced. This change now also sets some metadat to an item, but only if the item is not in state "confirmed". Confirmed items are not touched, but the metadata of the files is updated. --- .../scala/docspell/joex/JoexAppImpl.scala | 2 +- .../docspell/joex/process/LinkProposal.scala | 10 +++++- .../docspell/joex/process/ProcessItem.scala | 4 +-- .../docspell/joex/process/ReProcessItem.scala | 31 ++++++++++++++----- .../docspell/joex/process/SetGivenData.scala | 19 ++++++++---- .../src/main/resources/docspell-openapi.yml | 6 +++- 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 69a48906..c98d95d5 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -113,7 +113,7 @@ object JoexAppImpl { .withTask( JobTask.json( ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, analyser, regexNer), + ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer), ReProcessItem.onCancel[F] ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala index 6fa15978..6d0c8ac0 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -10,11 +10,19 @@ import docspell.store.records.RItem object LinkProposal { - def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def onlyNew[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = if (data.item.state.isValid) Task .log[F, ProcessItemArgs](_.debug(s"Not linking proposals on existing item")) .map(_ => data) + else + LinkProposal[F](data) + + def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + if (data.item.state == ItemState.Confirmed) + Task + .log[F, ProcessItemArgs](_.debug(s"Not linking proposals on confirmed item")) + .map(_ => data) else Task { ctx => val proposals = data.finalProposals diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 1ba548de..f3fd1862 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -22,8 +22,8 @@ object ProcessItem { ExtractArchive(item) .flatMap(Task.setProgress(20)) .flatMap(processAttachments0(cfg, fts, analyser, regexNer, (40, 60, 80))) - .flatMap(LinkProposal[F]) - .flatMap(SetGivenData[F](itemOps)) + .flatMap(LinkProposal.onlyNew[F]) + .flatMap(SetGivenData.onlyNew[F](itemOps)) .flatMap(Task.setProgress(99)) .flatMap(RemoveEmptyItem(itemOps)) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index e4e40f49..ae9911d1 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -5,6 +5,7 @@ import cats.effect._ import cats.implicits._ import docspell.analysis.TextAnalyser +import docspell.backend.ops.OItem import docspell.common._ import docspell.ftsclient.FtsClient import docspell.joex.Config @@ -22,12 +23,17 @@ object ReProcessItem { def apply[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, fts: FtsClient[F], + itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F] ): Task[F, Args, Unit] = - loadItem[F] - .flatMap(safeProcess[F](cfg, fts, analyser, regexNer)) - .map(_ => ()) + Task + .log[F, Args](_.info("===== Start reprocessing ======")) + .flatMap(_ => + loadItem[F] + .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer)) + .map(_ => ()) + ) def onCancel[F[_]]: Task[F, Args, Unit] = logWarn("Now cancelling re-processing.") @@ -58,6 +64,11 @@ object ReProcessItem { a.copy(fileId = src.fileId, name = src.name) } ) + _ <- OptionT.liftF( + ctx.logger.debug( + s"Loaded item and ${attachSrc.size} attachments to reprocess" + ) + ) } yield ItemData( item, attachSrc, @@ -76,6 +87,7 @@ object ReProcessItem { def processFiles[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, fts: FtsClient[F], + itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], data: ItemData @@ -89,9 +101,9 @@ object ReProcessItem { data.item.cid, args.itemId.some, lang, - None, //direction - "", //source-id - None, //folder + None, //direction + data.item.source, //source-id + None, //folder Seq.empty, false, None, @@ -103,6 +115,8 @@ object ReProcessItem { getLanguage[F].flatMap { lang => ProcessItem .processAttachments[F](cfg, fts, analyser, regexNer)(data) + .flatMap(LinkProposal[F]) + .flatMap(SetGivenData[F](itemOps)) .contramap[Args](convertArgs(lang)) } } @@ -121,12 +135,13 @@ object ReProcessItem { def safeProcess[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, fts: FtsClient[F], + itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - processFiles[F](cfg, fts, analyser, regexNer, data).attempt + processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt .flatMap({ case Right(d) => Task.pure(d) @@ -136,7 +151,7 @@ object ReProcessItem { ).andThen(_ => Sync[F].raiseError(ex)) }) case false => - processFiles[F](cfg, fts, analyser, regexNer, data) + processFiles[F](cfg, fts, itemOps, analyser, regexNer, data) } private def logWarn[F[_]](msg: => String): Task[F, Args, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala index b668dbe9..5d1c6038 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -8,13 +8,20 @@ import docspell.common._ import docspell.joex.scheduler.Task object SetGivenData { + type Args = ProcessItemArgs - def apply[F[_]: Sync]( - ops: OItem[F] - )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def onlyNew[F[_]: Sync](ops: OItem[F])(data: ItemData): Task[F, Args, ItemData] = if (data.item.state.isValid) Task - .log[F, ProcessItemArgs](_.debug(s"Not setting data on existing item")) + .log[F, Args](_.debug(s"Not setting data on existing item")) + .map(_ => data) + else + SetGivenData[F](ops)(data) + + def apply[F[_]: Sync](ops: OItem[F])(data: ItemData): Task[F, Args, ItemData] = + if (data.item.state == ItemState.Confirmed) + Task + .log[F, Args](_.debug(s"Not setting data on confirmed item")) .map(_ => data) else setFolder(data, ops).flatMap(d => setTags[F](d, ops)) @@ -22,7 +29,7 @@ object SetGivenData { private def setFolder[F[_]: Sync]( data: ItemData, ops: OItem[F] - ): Task[F, ProcessItemArgs, ItemData] = + ): Task[F, Args, ItemData] = Task { ctx => val itemId = data.item.id val folderId = ctx.args.meta.folderId @@ -41,7 +48,7 @@ object SetGivenData { private def setTags[F[_]: Sync]( data: ItemData, ops: OItem[F] - ): Task[F, ProcessItemArgs, ItemData] = + ): Task[F, Args, ItemData] = Task { ctx => val itemId = data.item.id val collective = ctx.args.meta.collective diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 41494740..e144ca07 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2113,7 +2113,11 @@ paths: summary: Start reprocessing the files of the item. description: | This submits a job that will re-process the files (either all - or the ones specified) of the item and replace the metadata. + or the ones specified) of the item and replace their metadata. + + If the item is not in "confirmed" state, its associated metada + is also updated. Otherwise only the file metadata is updated + (like extracted text etc). security: - authTokenHeader: [] parameters: From 76f5ab6c6831efecb07c317b4cea9eec6a401671 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 11 Mar 2021 23:13:34 +0100 Subject: [PATCH 2/4] Allow to reprocess single and multiple items in the ui --- .../src/main/resources/docspell-openapi.yml | 5 +- modules/webapp/src/main/elm/Api.elm | 31 ++++ .../webapp/src/main/elm/Comp/ConfirmModal.elm | 76 ++++++++++ .../src/main/elm/Comp/ItemDetail/Model.elm | 20 ++- .../elm/Comp/ItemDetail/SingleAttachment.elm | 26 +++- .../src/main/elm/Comp/ItemDetail/Update.elm | 133 ++++++++++++++---- .../src/main/elm/Comp/ItemDetail/View2.elm | 28 +++- .../webapp/src/main/elm/Page/Home/Data.elm | 11 +- .../webapp/src/main/elm/Page/Home/Update.elm | 108 +++++++++++--- .../webapp/src/main/elm/Page/Home/View2.elm | 25 +++- modules/webapp/src/main/elm/Styles.elm | 2 +- 11 files changed, 382 insertions(+), 83 deletions(-) create mode 100644 modules/webapp/src/main/elm/Comp/ConfirmModal.elm diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index e144ca07..fdb8488e 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2117,7 +2117,7 @@ paths: If the item is not in "confirmed" state, its associated metada is also updated. Otherwise only the file metadata is updated - (like extracted text etc). + (text analysis). security: - authTokenHeader: [] parameters: @@ -2519,7 +2519,8 @@ paths: description: | Given a list of item-ids, submits all these items for reprocessing. All attachments of these items will be - reprocessed. Item metadata is not changed. + reprocessed. Item metadata may be changed if an item is not + confirmed. Confirmed items are not changed. security: - authTokenHeader: [] requestBody: diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 30202fd6..b7869df7 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -89,6 +89,8 @@ module Api exposing , register , removeMember , removeTagsMultiple + , reprocessItem + , reprocessMultiple , sendMail , setAttachmentName , setCollectiveSettings @@ -1423,6 +1425,20 @@ getJobQueueStateTask flags = --- Item (Mulit Edit) +reprocessMultiple : + Flags + -> Set String + -> (Result Http.Error BasicResult -> msg) + -> Cmd msg +reprocessMultiple flags items receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/items/reprocess" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.IdList.encode (Set.toList items |> IdList)) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + confirmMultiple : Flags -> Set String @@ -1637,6 +1653,21 @@ deleteAllItems flags ids receive = --- Item +reprocessItem : + Flags + -> String + -> List String + -> (Result Http.Error BasicResult -> msg) + -> Cmd msg +reprocessItem flags itemId attachIds receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ itemId ++ "/reprocess" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.IdList.encode (IdList attachIds)) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + attachmentPreviewURL : String -> String attachmentPreviewURL id = "/api/v1/sec/attachment/" ++ id ++ "/preview?withFallback=true" diff --git a/modules/webapp/src/main/elm/Comp/ConfirmModal.elm b/modules/webapp/src/main/elm/Comp/ConfirmModal.elm new file mode 100644 index 00000000..10b5a3ad --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ConfirmModal.elm @@ -0,0 +1,76 @@ +module Comp.ConfirmModal exposing + ( Settings + , defaultSettings + , view + ) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Styles as S + + +type alias Settings msg = + { enabled : Bool + , extraClass : String + , headerIcon : String + , headerClass : String + , confirmText : String + , cancelText : String + , message : String + , confirm : msg + , cancel : msg + } + + +defaultSettings : msg -> msg -> String -> Settings msg +defaultSettings confirm cancel confirmMsg = + { enabled = True + , extraClass = "" + , headerIcon = "fa fa-exclamation-circle mr-3" + , headerClass = "text-2xl font-bold text-center w-full" + , confirmText = "Ok" + , cancelText = "Cancel" + , message = confirmMsg + , confirm = confirm + , cancel = cancel + } + + +view : Settings msg -> Html msg +view settings = + div + [ class S.dimmer + , class settings.extraClass + , classList + [ ( "hidden", not settings.enabled ) + ] + ] + [ div [ class settings.headerClass ] + [ i + [ class settings.headerIcon + , class "text-gray-200 font-semibold" + , classList [ ( "hidden", settings.headerClass == "" ) ] + ] + [] + , span [ class "text-gray-200 font-semibold" ] + [ text settings.message + ] + ] + , div [ class "flex flex-row space-x-2 text-xs mt-2" ] + [ a + [ class (S.primaryButton ++ "block font-semibold") + , href "#" + , onClick settings.confirm + ] + [ text settings.confirmText + ] + , a + [ class (S.secondaryButton ++ "block font-semibold") + , href "#" + , onClick settings.cancel + ] + [ text settings.cancelText + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm index 34e17c58..74a46d5f 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm @@ -28,6 +28,7 @@ import Api.Model.SentMails exposing (SentMails) import Api.Model.Tag exposing (Tag) import Api.Model.TagList exposing (TagList) import Comp.AttachmentMeta +import Comp.ConfirmModal import Comp.CustomFieldMultiInput import Comp.DatePicker import Comp.DetailEdit @@ -72,7 +73,7 @@ type alias Model = , nameSaveThrottle : Throttle Msg , notesModel : Maybe String , notesField : NotesField - , deleteItemConfirm : Comp.YesNoDimmer.Model + , itemModal : Maybe (Comp.ConfirmModal.Settings Msg) , itemDatePicker : DatePicker , itemDate : Maybe Int , itemProposals : ItemProposals @@ -87,7 +88,7 @@ type alias Model = , attachMeta : Dict String Comp.AttachmentMeta.Model , attachMetaOpen : Bool , pdfNativeView : Maybe Bool - , deleteAttachConfirm : Comp.YesNoDimmer.Model + , attachModal : Maybe (Comp.ConfirmModal.Settings Msg) , addFilesOpen : Bool , addFilesModel : Comp.Dropzone.Model , selectedFiles : List File @@ -180,7 +181,7 @@ emptyModel = , nameSaveThrottle = Throttle.create 1 , notesModel = Nothing , notesField = ViewNotes - , deleteItemConfirm = Comp.YesNoDimmer.emptyModel + , itemModal = Nothing , itemDatePicker = Comp.DatePicker.emptyModel , itemDate = Nothing , itemProposals = Api.Model.ItemProposals.empty @@ -195,7 +196,7 @@ emptyModel = , attachMeta = Dict.empty , attachMetaOpen = False , pdfNativeView = Nothing - , deleteAttachConfirm = Comp.YesNoDimmer.emptyModel + , attachModal = Nothing , addFilesOpen = False , addFilesModel = Comp.Dropzone.init [] , selectedFiles = [] @@ -247,7 +248,8 @@ type Msg | SetDueDateSuggestion Int | ItemDatePickerMsg Comp.DatePicker.Msg | DueDatePickerMsg Comp.DatePicker.Msg - | DeleteItemConfirm Comp.YesNoDimmer.Msg + | DeleteItemConfirmed + | ItemModalCancelled | RequestDelete | SaveResp (Result Http.Error BasicResult) | DeleteResp (Result Http.Error BasicResult) @@ -265,7 +267,8 @@ type Msg | AttachMetaMsg String Comp.AttachmentMeta.Msg | TogglePdfNativeView Bool | RequestDeleteAttachment String - | DeleteAttachConfirm String Comp.YesNoDimmer.Msg + | DeleteAttachConfirmed String + | AttachModalCancelled | DeleteAttachResp (Result Http.Error BasicResult) | AddFilesToggle | AddFilesMsg Comp.Dropzone.Msg @@ -304,6 +307,11 @@ type Msg | ToggleAttachmentDropdown | ToggleAkkordionTab String | ToggleOpenAllAkkordionTabs + | RequestReprocessFile String + | ReprocessFileConfirmed String + | ReprocessFileResp (Result Http.Error BasicResult) + | RequestReprocessItem + | ReprocessItemConfirmed type SaveNameState diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/SingleAttachment.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/SingleAttachment.elm index 2c8d3efc..7d3cd714 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/SingleAttachment.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/SingleAttachment.elm @@ -3,6 +3,7 @@ module Comp.ItemDetail.SingleAttachment exposing (view) import Api import Api.Model.Attachment exposing (Attachment) import Comp.AttachmentMeta +import Comp.ConfirmModal import Comp.ItemDetail.Model exposing ( Model @@ -11,7 +12,6 @@ import Comp.ItemDetail.Model , SaveNameState(..) ) import Comp.MenuBar as MB -import Comp.YesNoDimmer import Data.UiSettings exposing (UiSettings) import Dict import Html exposing (..) @@ -37,12 +37,7 @@ view settings model pos attach = [ ( "hidden", not (attachmentVisible model pos) ) ] ] - [ Html.map (DeleteAttachConfirm attach.id) - (Comp.YesNoDimmer.viewN - True - (Comp.YesNoDimmer.defaultSettings2 "Really delete this file?") - model.deleteAttachConfirm - ) + [ renderModal model , div [ class "flex flex-row px-2 py-2 text-sm" , class S.border @@ -213,6 +208,13 @@ attachHeader settings model _ attach = , href "#" ] } + , { icon = "fa fa-redo-alt" + , label = "Re-process this file" + , attrs = + [ onClick (RequestReprocessFile attach.id) + , href "#" + ] + } , { icon = "fa fa-trash" , label = "Delete this file" , attrs = @@ -344,3 +346,13 @@ menuItem model pos attach = |> text ] ] + + +renderModal : Model -> Html Msg +renderModal model = + case model.attachModal of + Just confirmModal -> + Comp.ConfirmModal.view confirmModal + + Nothing -> + span [ class "hidden" ] [] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm index 0dcc4043..99709b53 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm @@ -16,6 +16,7 @@ import Api.Model.ReferenceList exposing (ReferenceList) import Api.Model.Tag exposing (Tag) import Browser.Navigation as Nav import Comp.AttachmentMeta +import Comp.ConfirmModal import Comp.CustomFieldMultiInput import Comp.DatePicker import Comp.DetailEdit @@ -43,7 +44,6 @@ import Comp.MarkdownInput import Comp.OrgForm import Comp.PersonForm import Comp.SentMails -import Comp.YesNoDimmer import Data.CustomFieldChange exposing (CustomFieldChange(..)) import Data.Direction import Data.Fields exposing (Field) @@ -532,22 +532,28 @@ update key flags inav settings msg model = RemoveDueDate -> resultModelCmd ( { model | dueDate = Nothing }, setDueDate flags model Nothing ) - DeleteItemConfirm m -> + DeleteItemConfirmed -> let - ( cm, confirmed ) = - Comp.YesNoDimmer.update m model.deleteItemConfirm - cmd = - if confirmed then - Api.deleteItem flags model.item.id DeleteResp - - else - Cmd.none + Api.deleteItem flags model.item.id DeleteResp in - resultModelCmd ( { model | deleteItemConfirm = cm }, cmd ) + resultModelCmd ( { model | itemModal = Nothing }, cmd ) + + ItemModalCancelled -> + resultModel { model | itemModal = Nothing } RequestDelete -> - update key flags inav settings (DeleteItemConfirm Comp.YesNoDimmer.activate) model + let + confirmMsg = + "Really delete this item? This cannot be undone." + + confirm = + Comp.ConfirmModal.defaultSettings + DeleteItemConfirmed + ItemModalCancelled + confirmMsg + in + resultModel { model | itemModal = Just confirm } SetCorrOrgSuggestion idname -> resultModelCmd ( model, setCorrOrg flags model (Just idname) ) @@ -913,19 +919,15 @@ update key flags inav settings msg model = , attachmentDropdownOpen = False } - DeleteAttachConfirm attachId lmsg -> + DeleteAttachConfirmed attachId -> let - ( cm, confirmed ) = - Comp.YesNoDimmer.update lmsg model.deleteAttachConfirm - cmd = - if confirmed then - Api.deleteAttachment flags attachId DeleteAttachResp - - else - Cmd.none + Api.deleteAttachment flags attachId DeleteAttachResp in - resultModelCmd ( { model | deleteAttachConfirm = cm }, cmd ) + resultModelCmd ( { model | attachModal = Nothing }, cmd ) + + AttachModalCancelled -> + resultModel { model | attachModal = Nothing } DeleteAttachResp (Ok res) -> if res.success then @@ -938,12 +940,20 @@ update key flags inav settings msg model = resultModel model RequestDeleteAttachment id -> - update key - flags - inav - settings - (DeleteAttachConfirm id Comp.YesNoDimmer.activate) - { model | attachmentDropdownOpen = False } + let + confirmModal = + Comp.ConfirmModal.defaultSettings + (DeleteAttachConfirmed id) + AttachModalCancelled + "Really delete this file?" + + model_ = + { model + | attachmentDropdownOpen = False + , attachModal = Just confirmModal + } + in + resultModel model_ AddFilesToggle -> resultModel @@ -1508,6 +1518,73 @@ update key flags inav settings msg model = in resultModel { model | editMenuTabsOpen = next } + RequestReprocessFile id -> + let + confirmMsg = + if model.item.state == "created" then + "Reprocessing this file may change metadata of " + ++ "this item, since it is unconfirmed. Do you want to proceed?" + + else + "Reprocessing this file will not change metadata of " + ++ "this item, since it has been confirmed. Do you want to proceed?" + + confirmModal = + Comp.ConfirmModal.defaultSettings + (ReprocessFileConfirmed id) + AttachModalCancelled + confirmMsg + + model_ = + { model + | attachmentDropdownOpen = False + , attachModal = Just confirmModal + } + in + resultModel model_ + + ReprocessFileConfirmed id -> + let + cmd = + Api.reprocessItem flags model.item.id [ id ] ReprocessFileResp + in + resultModelCmd ( { model | attachModal = Nothing }, cmd ) + + ReprocessFileResp _ -> + resultModel model + + RequestReprocessItem -> + let + confirmMsg = + if model.item.state == "created" then + "Reprocessing this item may change its metadata, " + ++ "since it is unconfirmed. Do you want to proceed?" + + else + "Reprocessing this item will not change its metadata, " + ++ "since it has been confirmed. Do you want to proceed?" + + confirmModal = + Comp.ConfirmModal.defaultSettings + ReprocessItemConfirmed + ItemModalCancelled + confirmMsg + + model_ = + { model + | attachmentDropdownOpen = False + , itemModal = Just confirmModal + } + in + resultModel model_ + + ReprocessItemConfirmed -> + let + cmd = + Api.reprocessItem flags model.item.id [] ReprocessFileResp + in + resultModelCmd ( { model | itemModal = Nothing }, cmd ) + --- Helper diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm index 486566c4..377f0b16 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm @@ -1,6 +1,7 @@ module Comp.ItemDetail.View2 exposing (view) import Comp.Basic as B +import Comp.ConfirmModal import Comp.DetailEdit import Comp.ItemDetail.AddFilesForm import Comp.ItemDetail.ItemInfoHeader @@ -16,7 +17,6 @@ import Comp.ItemDetail.SingleAttachment import Comp.ItemMail import Comp.MenuBar as MB import Comp.SentMails -import Comp.YesNoDimmer import Data.Icons as Icons import Data.ItemNav exposing (ItemNav) import Data.UiSettings exposing (UiSettings) @@ -34,15 +34,20 @@ view inav settings model = [ header settings model , menuBar inav settings model , body inav settings model - , Html.map DeleteItemConfirm - (Comp.YesNoDimmer.viewN - True - (Comp.YesNoDimmer.defaultSettings2 "Really delete the complete item?") - model.deleteItemConfirm - ) + , itemModal model ] +itemModal : Model -> Html Msg +itemModal model = + case model.itemModal of + Just confirm -> + Comp.ConfirmModal.view confirm + + Nothing -> + span [ class "hidden" ] [] + + header : UiSettings -> Model -> Html Msg header settings model = div [ class "my-3" ] @@ -166,6 +171,15 @@ menuBar inav settings model = ] [ i [ class "fa fa-eye-slash font-thin" ] [] ] + , MB.CustomElement <| + a + [ class S.secondaryBasicButton + , href "#" + , onClick RequestReprocessItem + , title "Reprocess this item" + ] + [ i [ class "fa fa-redo" ] [] + ] , MB.CustomElement <| a [ class S.deleteButton diff --git a/modules/webapp/src/main/elm/Page/Home/Data.elm b/modules/webapp/src/main/elm/Page/Home/Data.elm index 7e6246cb..6112de60 100644 --- a/modules/webapp/src/main/elm/Page/Home/Data.elm +++ b/modules/webapp/src/main/elm/Page/Home/Data.elm @@ -22,6 +22,7 @@ import Api.Model.BasicResult exposing (BasicResult) import Api.Model.ItemLightList exposing (ItemLightList) import Api.Model.SearchStats exposing (SearchStats) import Browser.Dom as Dom +import Comp.ConfirmModal import Comp.FixedDropdown import Comp.ItemCardList import Comp.ItemDetail.FormChange exposing (FormChange) @@ -64,7 +65,7 @@ type alias Model = type alias SelectViewModel = { ids : Set String , action : SelectActionMode - , deleteAllConfirm : Comp.YesNoDimmer.Model + , confirmModal : Maybe (Comp.ConfirmModal.Settings Msg) , editModel : Comp.ItemDetail.MultiEditMenu.Model , saveNameState : SaveNameState , saveCustomFieldState : Set String @@ -75,7 +76,7 @@ initSelectViewModel : SelectViewModel initSelectViewModel = { ids = Set.empty , action = NoneAction - , deleteAllConfirm = Comp.YesNoDimmer.initActive + , confirmModal = Nothing , editModel = Comp.ItemDetail.MultiEditMenu.init , saveNameState = SaveSuccess , saveCustomFieldState = Set.empty @@ -187,7 +188,8 @@ type Msg | SelectAllItems | SelectNoItems | RequestDeleteSelected - | DeleteSelectedConfirmMsg Comp.YesNoDimmer.Msg + | DeleteSelectedConfirmed + | CloseConfirmModal | EditSelectedItems | EditMenuMsg Comp.ItemDetail.MultiEditMenu.Msg | MultiUpdateResp FormChange (Result Http.Error BasicResult) @@ -199,6 +201,8 @@ type Msg | TogglePreviewFullWidth | PowerSearchMsg Comp.PowerSearchInput.Msg | KeyUpPowerSearchbarMsg (Maybe KeyCode) + | RequestReprocessSelected + | ReprocessSelectedConfirmed type SearchType @@ -210,6 +214,7 @@ type SelectActionMode = NoneAction | DeleteSelected | EditSelected + | ReprocessSelected type alias SearchParam = diff --git a/modules/webapp/src/main/elm/Page/Home/Update.elm b/modules/webapp/src/main/elm/Page/Home/Update.elm index 9d8efee9..04ad7529 100644 --- a/modules/webapp/src/main/elm/Page/Home/Update.elm +++ b/modules/webapp/src/main/elm/Page/Home/Update.elm @@ -3,6 +3,7 @@ module Page.Home.Update exposing (update) import Api import Api.Model.ItemLightList exposing (ItemLightList) import Browser.Navigation as Nav +import Comp.ConfirmModal import Comp.FixedDropdown import Comp.ItemCardList import Comp.ItemDetail.FormChange exposing (FormChange(..)) @@ -10,7 +11,6 @@ import Comp.ItemDetail.MultiEditMenu exposing (SaveNameState(..)) import Comp.LinkTarget exposing (LinkTarget) import Comp.PowerSearchInput import Comp.SearchMenu -import Comp.YesNoDimmer import Data.Flags exposing (Flags) import Data.ItemQuery as Q import Data.ItemSelection @@ -358,34 +358,20 @@ update mId key flags settings msg model = _ -> noSub ( model, Cmd.none ) - DeleteSelectedConfirmMsg lmsg -> + DeleteSelectedConfirmed -> case model.viewMode of SelectView svm -> let - ( confirmModel, confirmed ) = - Comp.YesNoDimmer.update lmsg svm.deleteAllConfirm - cmd = - if confirmed then - Api.deleteAllItems flags svm.ids DeleteAllResp - - else - Cmd.none - - act = - if confirmModel.active || confirmed then - DeleteSelected - - else - NoneAction + Api.deleteAllItems flags svm.ids DeleteAllResp in noSub ( { model | viewMode = SelectView { svm - | deleteAllConfirm = confirmModel - , action = act + | confirmModal = Nothing + , action = DeleteSelected } } , cmd @@ -416,6 +402,74 @@ update mId key flags settings msg model = DeleteAllResp (Err _) -> noSub ( model, Cmd.none ) + RequestReprocessSelected -> + case model.viewMode of + SelectView svm -> + if svm.ids == Set.empty then + noSub ( model, Cmd.none ) + + else + let + lmsg = + Comp.ConfirmModal.defaultSettings + ReprocessSelectedConfirmed + CloseConfirmModal + "Really reprocess all selected items? Metadata of unconfirmed items may change." + + model_ = + { model + | viewMode = + SelectView + { svm + | action = ReprocessSelected + , confirmModal = Just lmsg + } + } + in + noSub ( model_, Cmd.none ) + + _ -> + noSub ( model, Cmd.none ) + + CloseConfirmModal -> + case model.viewMode of + SelectView svm -> + noSub + ( { model + | viewMode = SelectView { svm | confirmModal = Nothing, action = NoneAction } + } + , Cmd.none + ) + + _ -> + noSub ( model, Cmd.none ) + + ReprocessSelectedConfirmed -> + case model.viewMode of + SelectView svm -> + if svm.ids == Set.empty then + noSub ( model, Cmd.none ) + + else + let + cmd = + Api.reprocessMultiple flags svm.ids DeleteAllResp + in + noSub + ( { model + | viewMode = + SelectView + { svm + | confirmModal = Nothing + , action = ReprocessSelected + } + } + , cmd + ) + + _ -> + noSub ( model, Cmd.none ) + RequestDeleteSelected -> case model.viewMode of SelectView svm -> @@ -425,12 +479,22 @@ update mId key flags settings msg model = else let lmsg = - DeleteSelectedConfirmMsg Comp.YesNoDimmer.activate + Comp.ConfirmModal.defaultSettings + DeleteSelectedConfirmed + CloseConfirmModal + "Really delete all selected items?" model_ = - { model | viewMode = SelectView { svm | action = DeleteSelected } } + { model + | viewMode = + SelectView + { svm + | action = DeleteSelected + , confirmModal = Just lmsg + } + } in - update mId key flags settings lmsg model_ + noSub ( model_, Cmd.none ) _ -> noSub ( model, Cmd.none ) diff --git a/modules/webapp/src/main/elm/Page/Home/View2.elm b/modules/webapp/src/main/elm/Page/Home/View2.elm index 4b6d6f8e..1453057d 100644 --- a/modules/webapp/src/main/elm/Page/Home/View2.elm +++ b/modules/webapp/src/main/elm/Page/Home/View2.elm @@ -1,6 +1,7 @@ module Page.Home.View2 exposing (viewContent, viewSidebar) import Comp.Basic as B +import Comp.ConfirmModal import Comp.ItemCardList import Comp.MenuBar as MB import Comp.PowerSearchInput @@ -67,13 +68,13 @@ deleteSelectedDimmer model = in case model.viewMode of SelectView svm -> - [ Html.map DeleteSelectedConfirmMsg - (Comp.YesNoDimmer.viewN - (selectAction == DeleteSelected) - deleteAllDimmer - svm.deleteAllConfirm - ) - ] + case svm.confirmModal of + Just confirm -> + [ Comp.ConfirmModal.view confirm + ] + + Nothing -> + [] _ -> [] @@ -219,6 +220,16 @@ editMenuBar model svm = , ( "bg-gray-200 dark:bg-bluegray-600", svm.action == EditSelected ) ] } + , MB.CustomButton + { tagger = RequestReprocessSelected + , label = "" + , icon = Just "fa fa-redo" + , title = "Reprocess " ++ selectCount ++ " selected items" + , inputClass = + [ ( btnStyle, True ) + , ( "bg-gray-200 dark:bg-bluegray-600", svm.action == ReprocessSelected ) + ] + } , MB.CustomButton { tagger = RequestDeleteSelected , label = "" diff --git a/modules/webapp/src/main/elm/Styles.elm b/modules/webapp/src/main/elm/Styles.elm index 2d917c49..ea5c5f94 100644 --- a/modules/webapp/src/main/elm/Styles.elm +++ b/modules/webapp/src/main/elm/Styles.elm @@ -313,7 +313,7 @@ editLinkTableCellStyle = dimmer : String dimmer = - " absolute top-0 left-0 w-full h-full bg-black bg-opacity-90 dark:bg-bluegray-900 dark:bg-opacity-90 z-50 flex flex-col items-center justify-center px-4 py-2 " + " absolute top-0 left-0 w-full h-full bg-black bg-opacity-90 dark:bg-bluegray-900 dark:bg-opacity-90 z-50 flex flex-col items-center justify-center px-4 md:px-8 py-2 " dimmerLight : String From a7ee0aa08b0f522173d5cabf904827fa9cff3015 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 12 Mar 2021 00:33:51 +0100 Subject: [PATCH 3/4] Add a flag to processing task to distinguish re-/processing --- .../src/main/scala/docspell/backend/ops/OUpload.scala | 3 ++- .../src/main/scala/docspell/common/ProcessItemArgs.scala | 7 ++++++- .../main/scala/docspell/joex/process/ReProcessItem.scala | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index 12254f08..b87f420b 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -145,7 +145,8 @@ object OUpload { data.meta.validFileTypes, data.meta.skipDuplicates, data.meta.fileFilter.some, - data.meta.tags.some + data.meta.tags.some, + false ) args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index aba6974e..87995da0 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -13,6 +13,8 @@ import io.circe.generic.semiauto._ * * If the `itemId' is set to some value, the item is tried to load to * ammend with the given files. Otherwise a new item is created. + * + * It is also re-used by the 'ReProcessItem' task. */ case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { @@ -24,6 +26,8 @@ case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { case _ => s"${files.size} files from ${meta.sourceAbbrev}" } + def isNormalProcessing: Boolean = + !meta.reprocess } object ProcessItemArgs { @@ -40,7 +44,8 @@ object ProcessItemArgs { validFileTypes: Seq[MimeType], skipDuplicate: Boolean, fileFilter: Option[Glob], - tags: Option[List[String]] + tags: Option[List[String]], + reprocess: Boolean ) object ProcessMeta { diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index ae9911d1..2f0188fc 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -107,7 +107,8 @@ object ReProcessItem { Seq.empty, false, None, - None + None, + true ), Nil ).pure[F] From f8bd42e5bd977e0b6ebc6d3366a2525391360a11 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 12 Mar 2021 00:45:28 +0100 Subject: [PATCH 4/4] Redo pdf conversion and text extraction on reprocess When processing a new file conversion and text extraction is skipped if detected to be already done. This prevents running expensive tasks again after restarting/retrying. When explicitely reprocessing a file, these tasks should run again and replace the existing results. --- .../src/main/scala/docspell/joex/process/ConvertPdf.scala | 4 ++-- .../src/main/scala/docspell/joex/process/TextExtraction.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala index 57292563..84828e19 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala @@ -40,14 +40,14 @@ object ConvertPdf { Task { ctx => def convert(ra: RAttachment): F[(RAttachment, Option[RAttachmentMeta])] = isConverted(ctx)(ra).flatMap { - case true => + case true if ctx.args.isNormalProcessing => ctx.logger.info( s"Conversion to pdf already done for attachment ${ra.name}." ) *> ctx.store .transact(RAttachmentMeta.findById(ra.id)) .map(rmOpt => (ra, rmOpt)) - case false => + case _ => findMime(ctx)(ra).flatMap(m => convertSafe(cfg, JsoupSanitizer.clean, ctx, item)(ra, m) ) diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index fcdd6f98..2dcc4d31 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -84,10 +84,10 @@ object TextExtraction { val rm = item.findOrCreate(ra.id, lang) rm.content match { - case Some(_) => + case Some(_) if ctx.args.isNormalProcessing => ctx.logger.info("TextExtraction skipped, since text is already available.") *> makeTextData((rm, Nil)).pure[F] - case None => + case _ => extractTextToMeta[F](ctx, cfg, lang, item)(ra) .map(makeTextData) }