From 4cba96f39081e6e4ac5ce1d663ebebe1e7804461 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 21 Jan 2021 21:05:28 +0100 Subject: [PATCH] Always return classifier results as suggestion The classifier results are spliced into the suggestion list at second place. When linking they are only used if nlp didn't find anything. --- .../scala/docspell/backend/ops/OItem.scala | 2 +- .../docspell/common/MetaProposalList.scala | 44 +++++++++++++------ .../common/MetaProposalListTest.scala | 31 +++++++++++++ .../joex/learn/LearnClassifierTask.scala | 1 + .../docspell/store/queries/QAttachment.scala | 2 +- .../src/main/elm/Comp/ItemDetail/View.elm | 13 +++--- 6 files changed, 71 insertions(+), 22 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index bcefe0e5..53acd38d 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -591,7 +591,7 @@ object OItem { for { itemIds <- store.transact(RItem.filterItems(items, collective)) results <- itemIds.traverse(item => deleteItem(item, collective)) - n = results.fold(0)(_ + _) + n = results.sum } yield n def getProposals(item: Ident, collective: Ident): F[MetaProposalList] = diff --git a/modules/common/src/main/scala/docspell/common/MetaProposalList.scala b/modules/common/src/main/scala/docspell/common/MetaProposalList.scala index d75693d8..04cedb30 100644 --- a/modules/common/src/main/scala/docspell/common/MetaProposalList.scala +++ b/modules/common/src/main/scala/docspell/common/MetaProposalList.scala @@ -45,6 +45,19 @@ case class MetaProposalList private (proposals: List[MetaProposal]) { def sortByWeights: MetaProposalList = change(_.sortByWeight) + + def insertSecond(ml: MetaProposalList): MetaProposalList = + MetaProposalList.flatten0( + Seq(this, ml), + (map, next) => + map.get(next.proposalType) match { + case Some(MetaProposal(mt, values)) => + val cand = NonEmptyList(values.head, next.values.toList ++ values.tail) + map.updated(next.proposalType, MetaProposal(mt, MetaProposal.flatten(cand))) + case None => + map.updated(next.proposalType, next) + } + ) } object MetaProposalList { @@ -74,20 +87,25 @@ object MetaProposalList { * is preserved and candidates of proposals are appended as given * by the order of the given `seq'. */ - def flatten(ml: Seq[MetaProposalList]): MetaProposalList = { - val init: Map[MetaProposalType, MetaProposal] = Map.empty - - def updateMap( - map: Map[MetaProposalType, MetaProposal], - mp: MetaProposal - ): Map[MetaProposalType, MetaProposal] = - map.get(mp.proposalType) match { - case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList)) - case None => map.updated(mp.proposalType, mp) - } - - val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(updateMap)) + def flatten(ml: Seq[MetaProposalList]): MetaProposalList = + flatten0( + ml, + (map, mp) => + map.get(mp.proposalType) match { + case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList)) + case None => map.updated(mp.proposalType, mp) + } + ) + private def flatten0( + ml: Seq[MetaProposalList], + merge: ( + Map[MetaProposalType, MetaProposal], + MetaProposal + ) => Map[MetaProposalType, MetaProposal] + ): MetaProposalList = { + val init = Map.empty[MetaProposalType, MetaProposal] + val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(merge)) fromMap(merged) } diff --git a/modules/common/src/test/scala/docspell/common/MetaProposalListTest.scala b/modules/common/src/test/scala/docspell/common/MetaProposalListTest.scala index 4b652f62..44a6cfc2 100644 --- a/modules/common/src/test/scala/docspell/common/MetaProposalListTest.scala +++ b/modules/common/src/test/scala/docspell/common/MetaProposalListTest.scala @@ -68,4 +68,35 @@ object MetaProposalListTest extends SimpleTestSuite { assertEquals(candidates.head, cand1) assertEquals(candidates.tail.head, cand2) } + + test("insert second") { + val cand1 = Candidate(IdRef(Ident.unsafe("123"), "name"), Set.empty) + val cand2 = Candidate(IdRef(Ident.unsafe("456"), "name"), Set.empty) + val cand3 = Candidate(IdRef(Ident.unsafe("789"), "name"), Set.empty) + val cand4 = Candidate(IdRef(Ident.unsafe("abc"), "name"), Set.empty) + val cand5 = Candidate(IdRef(Ident.unsafe("def"), "name"), Set.empty) + + val mpl1 = MetaProposalList + .of( + MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand1, cand2)), + MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand3)) + ) + + val mpl2 = MetaProposalList + .of( + MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand4)), + MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand5)) + ) + + val result = mpl1.insertSecond(mpl2) + assertEquals( + result, + MetaProposalList( + List( + MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand1, cand4, cand2)), + MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand3, cand5)) + ) + ) + ) + } } diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala index be3d7143..89d7886a 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala @@ -26,6 +26,7 @@ object LearnClassifierTask { ): Task[F, Args, Unit] = learnTags(cfg, analyser) .flatMap(_ => learnItemEntities(cfg, analyser)) + .flatMap(_ => Task(_ => Sync[F].delay(System.gc()))) private def learnItemEntities[F[_]: Sync: ContextShift]( cfg: Config.TextAnalysis, diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 89c11faf..1b6fa8ab 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -139,7 +139,7 @@ object QAttachment { mli <- qi.query[MetaProposalList].to[Vector] } yield MetaProposalList .flatten(mla) - .fillEmptyFrom(MetaProposalList.flatten(mli)) + .insertSecond(MetaProposalList.flatten(mli)) } def getAttachmentMeta( diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm index d9355d05..022ed1f4 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm @@ -958,7 +958,6 @@ renderSuggestions model mkName idnames tagger = ] , div [ class "menu" ] <| (idnames - |> List.take 5 |> List.map (\p -> a [ class "item", href "#", onClick (tagger p) ] [ text (mkName p) ]) ) ] @@ -969,7 +968,7 @@ renderOrgSuggestions : Model -> Html Msg renderOrgSuggestions model = renderSuggestions model .name - (List.take 5 model.itemProposals.corrOrg) + (List.take 6 model.itemProposals.corrOrg) SetCorrOrgSuggestion @@ -977,7 +976,7 @@ renderCorrPersonSuggestions : Model -> Html Msg renderCorrPersonSuggestions model = renderSuggestions model .name - (List.take 5 model.itemProposals.corrPerson) + (List.take 6 model.itemProposals.corrPerson) SetCorrPersonSuggestion @@ -985,7 +984,7 @@ renderConcPersonSuggestions : Model -> Html Msg renderConcPersonSuggestions model = renderSuggestions model .name - (List.take 5 model.itemProposals.concPerson) + (List.take 6 model.itemProposals.concPerson) SetConcPersonSuggestion @@ -993,7 +992,7 @@ renderConcEquipSuggestions : Model -> Html Msg renderConcEquipSuggestions model = renderSuggestions model .name - (List.take 5 model.itemProposals.concEquipment) + (List.take 6 model.itemProposals.concEquipment) SetConcEquipSuggestion @@ -1001,7 +1000,7 @@ renderItemDateSuggestions : Model -> Html Msg renderItemDateSuggestions model = renderSuggestions model Util.Time.formatDate - (List.take 5 model.itemProposals.itemDate) + (List.take 6 model.itemProposals.itemDate) SetItemDateSuggestion @@ -1009,7 +1008,7 @@ renderDueDateSuggestions : Model -> Html Msg renderDueDateSuggestions model = renderSuggestions model Util.Time.formatDate - (List.take 5 model.itemProposals.dueDate) + (List.take 6 model.itemProposals.dueDate) SetDueDateSuggestion