Always return classifier results as suggestion

The classifier results are spliced into the suggestion list at second
place. When linking they are only used if nlp didn't find anything.

modules/backend/src/main/scala/docspell/backend/ops/OItem.scala

@@ -591,7 +591,7 @@ object OItem {
           for {
             itemIds <- store.transact(RItem.filterItems(items, collective))
             results <- itemIds.traverse(item => deleteItem(item, collective))
-            n = results.fold(0)(_ + _)
+            n = results.sum
           } yield n
 
         def getProposals(item: Ident, collective: Ident): F[MetaProposalList] =

modules/common/src/main/scala/docspell/common/MetaProposalList.scala

@@ -45,6 +45,19 @@ case class MetaProposalList private (proposals: List[MetaProposal]) {
 
   def sortByWeights: MetaProposalList =
     change(_.sortByWeight)
+
+  def insertSecond(ml: MetaProposalList): MetaProposalList =
+    MetaProposalList.flatten0(
+      Seq(this, ml),
+      (map, next) =>
+        map.get(next.proposalType) match {
+          case Some(MetaProposal(mt, values)) =>
+            val cand = NonEmptyList(values.head, next.values.toList ++ values.tail)
+            map.updated(next.proposalType, MetaProposal(mt, MetaProposal.flatten(cand)))
+          case None =>
+            map.updated(next.proposalType, next)
+        }
+    )
 }
 
 object MetaProposalList {
@@ -74,20 +87,25 @@ object MetaProposalList {
     * is preserved and candidates of proposals are appended as given
     * by the order of the given `seq'.
     */
-  def flatten(ml: Seq[MetaProposalList]): MetaProposalList = {
+  def flatten(ml: Seq[MetaProposalList]): MetaProposalList =
-    val init: Map[MetaProposalType, MetaProposal] = Map.empty
+    flatten0(
-
+      ml,
-    def updateMap(
+      (map, mp) =>
-        map: Map[MetaProposalType, MetaProposal],
+        map.get(mp.proposalType) match {
-        mp: MetaProposal
+          case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList))
-    ): Map[MetaProposalType, MetaProposal] =
+          case None      => map.updated(mp.proposalType, mp)
-      map.get(mp.proposalType) match {
+        }
-        case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList))
+    )
-        case None      => map.updated(mp.proposalType, mp)
-      }
-
-    val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(updateMap))
 
+  private def flatten0(
+      ml: Seq[MetaProposalList],
+      merge: (
+          Map[MetaProposalType, MetaProposal],
+          MetaProposal
+      ) => Map[MetaProposalType, MetaProposal]
+  ): MetaProposalList = {
+    val init   = Map.empty[MetaProposalType, MetaProposal]
+    val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(merge))
     fromMap(merged)
   }
 

modules/common/src/test/scala/docspell/common/MetaProposalListTest.scala

@@ -68,4 +68,35 @@ object MetaProposalListTest extends SimpleTestSuite {
     assertEquals(candidates.head, cand1)
     assertEquals(candidates.tail.head, cand2)
   }
+
+  test("insert second") {
+    val cand1 = Candidate(IdRef(Ident.unsafe("123"), "name"), Set.empty)
+    val cand2 = Candidate(IdRef(Ident.unsafe("456"), "name"), Set.empty)
+    val cand3 = Candidate(IdRef(Ident.unsafe("789"), "name"), Set.empty)
+    val cand4 = Candidate(IdRef(Ident.unsafe("abc"), "name"), Set.empty)
+    val cand5 = Candidate(IdRef(Ident.unsafe("def"), "name"), Set.empty)
+
+    val mpl1 = MetaProposalList
+      .of(
+        MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand1, cand2)),
+        MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand3))
+      )
+
+    val mpl2 = MetaProposalList
+      .of(
+        MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand4)),
+        MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand5))
+      )
+
+    val result = mpl1.insertSecond(mpl2)
+    assertEquals(
+      result,
+      MetaProposalList(
+        List(
+          MetaProposal(MetaProposalType.CorrOrg, NonEmptyList.of(cand1, cand4, cand2)),
+          MetaProposal(MetaProposalType.ConcPerson, NonEmptyList.of(cand3, cand5))
+        )
+      )
+    )
+  }
 }

modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala

@@ -26,6 +26,7 @@ object LearnClassifierTask {
   ): Task[F, Args, Unit] =
     learnTags(cfg, analyser)
       .flatMap(_ => learnItemEntities(cfg, analyser))
+      .flatMap(_ => Task(_ => Sync[F].delay(System.gc())))
 
   private def learnItemEntities[F[_]: Sync: ContextShift](
       cfg: Config.TextAnalysis,

modules/store/src/main/scala/docspell/store/queries/QAttachment.scala

@@ -139,7 +139,7 @@ object QAttachment {
       mli <- qi.query[MetaProposalList].to[Vector]
     } yield MetaProposalList
       .flatten(mla)
-      .fillEmptyFrom(MetaProposalList.flatten(mli))
+      .insertSecond(MetaProposalList.flatten(mli))
   }
 
   def getAttachmentMeta(

modules/webapp/src/main/elm/Comp/ItemDetail/View.elm

@@ -958,7 +958,6 @@ renderSuggestions model mkName idnames tagger =
                 ]
             , div [ class "menu" ] <|
                 (idnames
-                    |> List.take 5
                     |> List.map (\p -> a [ class "item", href "#", onClick (tagger p) ] [ text (mkName p) ])
                 )
             ]
@@ -969,7 +968,7 @@ renderOrgSuggestions : Model -> Html Msg
 renderOrgSuggestions model =
     renderSuggestions model
         .name
-        (List.take 5 model.itemProposals.corrOrg)
+        (List.take 6 model.itemProposals.corrOrg)
         SetCorrOrgSuggestion
 
 
@@ -977,7 +976,7 @@ renderCorrPersonSuggestions : Model -> Html Msg
 renderCorrPersonSuggestions model =
     renderSuggestions model
         .name
-        (List.take 5 model.itemProposals.corrPerson)
+        (List.take 6 model.itemProposals.corrPerson)
         SetCorrPersonSuggestion
 
 
@@ -985,7 +984,7 @@ renderConcPersonSuggestions : Model -> Html Msg
 renderConcPersonSuggestions model =
     renderSuggestions model
         .name
-        (List.take 5 model.itemProposals.concPerson)
+        (List.take 6 model.itemProposals.concPerson)
         SetConcPersonSuggestion
 
 
@@ -993,7 +992,7 @@ renderConcEquipSuggestions : Model -> Html Msg
 renderConcEquipSuggestions model =
     renderSuggestions model
         .name
-        (List.take 5 model.itemProposals.concEquipment)
+        (List.take 6 model.itemProposals.concEquipment)
         SetConcEquipSuggestion
 
 
@@ -1001,7 +1000,7 @@ renderItemDateSuggestions : Model -> Html Msg
 renderItemDateSuggestions model =
     renderSuggestions model
         Util.Time.formatDate
-        (List.take 5 model.itemProposals.itemDate)
+        (List.take 6 model.itemProposals.itemDate)
         SetItemDateSuggestion
 
 
@@ -1009,7 +1008,7 @@ renderDueDateSuggestions : Model -> Html Msg
 renderDueDateSuggestions model =
     renderSuggestions model
         Util.Time.formatDate
-        (List.take 5 model.itemProposals.dueDate)
+        (List.take 6 model.itemProposals.dueDate)
         SetDueDateSuggestion