Add endpoint to search for items and return their tags

This is a more expensive query, since the tags must be resolved per
item. This is now implemented by doing additional queries while
caching each resolved tag.

modules/backend/src/main/scala/docspell/backend/ops/OItem.scala

@@ -15,6 +15,7 @@ import OItem.{
   Batch,
   ItemData,
   ListItem,
+  ListItemWithTags,
   Query
 }
 import bitpeace.{FileMeta, RangeDef}
@@ -27,6 +28,9 @@ trait OItem[F[_]] {
 
   def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
 
+  /** Same as `findItems` but does more queries per item to find all tags. */
+  def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
+
   def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
 
   def findAttachmentSource(
@@ -91,6 +95,9 @@ object OItem {
   type ListItem = QItem.ListItem
   val ListItem = QItem.ListItem
 
+  type ListItemWithTags = QItem.ListItemWithTags
+  val ListItemWithTags = QItem.ListItemWithTags
+
   type ItemData = QItem.ItemData
   val ItemData = QItem.ItemData
 
@@ -148,6 +155,12 @@ object OItem {
           .compile
           .toVector
 
+      def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
+        store
+          .transact(QItem.findItemsWithTags(q, batch).take(batch.limit.toLong))
+          .compile
+          .toVector
+
       def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
         store
           .transact(RAttachment.findByIdAndCollective(id, collective))

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -987,7 +987,31 @@ paths:
       summary: Search for items.
       description: |
         Search for items given a search form. The results are grouped
-        by month by default.
+        by month by default. Tags are *not* resolved! The results will
+        always contain an empty list for item tags. Use
+        `/searchWithTags` to also retrieve all tags of an item.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ItemSearch"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ItemLightList"
+  /sec/item/searchWithTags:
+    post:
+      tags: [ Item ]
+      summary: Search for items.
+      description: |
+        Search for items given a search form. The results are grouped
+        by month by default. For each item, its tags are also
+        returned. This uses more queries and is therefore slower.
       security:
         - authTokenHeader: []
       requestBody:
@@ -3188,6 +3212,7 @@ components:
         - date
         - source
         - fileCount
+        - tags
       properties:
         id:
           type: string
@@ -3221,6 +3246,10 @@ components:
         fileCount:
           type: integer
           format: int32
+        tags:
+          type: array
+          items:
+            $ref: "#/components/schemas/Tag"
     IdName:
       description: |
         The identifier and a human readable name of some entity.

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -135,6 +135,17 @@ trait Conversions {
     ItemLightList(gs)
   }
 
+  def mkItemListWithTags(v: Vector[OItem.ListItemWithTags]): ItemLightList = {
+    val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
+
+    def mkGroup(g: (String, Vector[OItem.ListItemWithTags])): ItemLightGroup =
+      ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList)
+
+    val gs =
+      groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
+    ItemLightList(gs)
+  }
+
   def mkItemLight(i: OItem.ListItem): ItemLight =
     ItemLight(
       i.id,
@@ -148,9 +159,13 @@ trait Conversions {
       i.corrPerson.map(mkIdName),
       i.concPerson.map(mkIdName),
       i.concEquip.map(mkIdName),
-      i.fileCount
+      i.fileCount,
+      Nil
     )
 
+  def mkItemLightWithTags(i: OItem.ListItemWithTags): ItemLight =
+    mkItemLight(i.item).copy(tags = i.tags.map(mkTag))
+
   // job
   def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
     def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = {

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -36,6 +36,19 @@ object ItemRoutes {
           resp <- Ok(Conversions.mkItemList(items))
         } yield resp
 
+      case req @ POST -> Root / "searchWithTags" =>
+        for {
+          mask <- req.as[ItemSearch]
+          _    <- logger.ftrace(s"Got search mask: $mask")
+          query = Conversions.mkQuery(mask, user.account.collective)
+          _ <- logger.ftrace(s"Running query: $query")
+          items <- backend.item.findItemsWithTags(
+            query,
+            Batch(mask.offset, mask.limit).restrictLimitTo(500)
+          )
+          resp <- Ok(Conversions.mkItemListWithTags(items))
+        } yield resp
+
       case GET -> Root / Ident(id) =>
         for {
           item <- backend.item.findItem(id, user.account.collective)

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -4,6 +4,7 @@ import bitpeace.FileMeta
 import cats.effect.Sync
 import cats.data.OptionT
 import cats.implicits._
+import cats.effect.concurrent.Ref
 import fs2.Stream
 import doobie._
 import doobie.implicits._
@@ -255,18 +256,10 @@ object QItem {
     ) ++
       fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++
       fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
-      fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson
+      fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++
-      .prefix("i")
+      fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++
-      .is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
+      fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++
-      fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg
+      fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1"))
-      .prefix("i")
-      .is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
-      fr"LEFT JOIN persons p1 ON" ++ IC.concPerson
-      .prefix("i")
-      .is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
-      fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
-      .prefix("i")
-      .is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
 
     // inclusive tags are AND-ed
     val tagSelectsIncl = q.tagsInclude
@@ -339,6 +332,43 @@ object QItem {
     frag.query[ListItem].stream
   }
 
+  case class ListItemWithTags(item: ListItem, tags: List[RTag])
+
+  /** Same as `findItems` but resolves the tags for each item. Note that
+    * this is implemented by running an additional query per item.
+    */
+  def findItemsWithTags(
+      q: Query,
+      batch: Batch
+  ): Stream[ConnectionIO, ListItemWithTags] = {
+    def findTag(
+        cache: Ref[ConnectionIO, Map[Ident, RTag]],
+        tagItem: RTagItem
+    ): ConnectionIO[Option[RTag]] =
+      for {
+        cc <- cache.get
+        fromCache = cc.get(tagItem.tagId)
+        orFromDB <-
+          if (fromCache.isDefined) fromCache.pure[ConnectionIO]
+          else RTag.findById(tagItem.tagId)
+        _ <-
+          if (fromCache.isDefined) ().pure[ConnectionIO]
+          else
+            orFromDB match {
+              case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
+              case None    => ().pure[ConnectionIO]
+            }
+      } yield orFromDB
+
+    for {
+      resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
+      item         <- findItems(q, batch)
+      tagItems     <- Stream.eval(RTagItem.findByItem(item.id))
+      tags         <- Stream.eval(tagItems.traverse(ti => findTag(resolvedTags, ti)))
+      ftags = tags.flatten.filter(t => t.collective == q.collective)
+    } yield ListItemWithTags(item, ftags.toList.sortBy(_.name))
+  }
+
   def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
     for {
       tn <- store.transact(RTagItem.deleteItemTags(itemId))

modules/store/src/main/scala/docspell/store/records/RTagItem.scala

@@ -38,4 +38,7 @@ object RTagItem {
       tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
       ins <- insertRows(table, all, tagFrag).update.run
     } yield ins
+
+  def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] =
+    selectSimple(all, table, itemId.is(item)).query[RTagItem].to[Vector]
 }