Improve querying item results

The join to retrieve the attachment count per item turned out to be
very expensive. And it is not used anymore for the result, only to
support fulltext search. It is now removed from the query. The
DISTINCT keyword is also removed, because it is not necessary and it
is expensive. With the DISTINCT removed, a new index (provided in the
previous commit) can now be used to avoid sorting items.

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -291,10 +291,13 @@ object OFulltext {
           for {
             items <- sqlResult
             ids    = items.map(a => ItemId[A].itemId(a))
+            idsNel = NonEmptyList.fromFoldable(ids)
             // must find all index results involving the items.
             // Currently there is one result per item + one result per
             // attachment
-            limit = items.map(a => ItemId[A].fileCount(a)).sum + items.size
+            limit <- idsNel
+              .map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds)))
+              .getOrElse(0.pure[F])
             ftsQ = fq.copy(items = ids.toSet, limit = limit)
             ftsR <- fts.search(ftsQ)
             ftsItems = ftsR.results.groupBy(_.itemId)
@@ -320,22 +323,19 @@ object OFulltext {
 
   trait ItemId[A] {
     def itemId(a: A): Ident
-
-    def fileCount(a: A): Int
   }
   object ItemId {
     def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev
 
-    def from[A](f: A => Ident, g: A => Int): ItemId[A] =
+    def from[A](f: A => Ident): ItemId[A] =
       new ItemId[A] {
         def itemId(a: A) = f(a)
-        def fileCount(a: A) = g(a)
       }
 
     implicit val listItemId: ItemId[ListItem] =
-      ItemId.from(_.id, _.fileCount)
+      ItemId.from(_.id)
 
     implicit val listItemWithTagsId: ItemId[ListItemWithTags] =
-      ItemId.from(_.item.id, _.item.fileCount)
+      ItemId.from(_.item.id)
   }
 }

modules/store/src/main/scala/docspell/store/queries/ListItem.scala

@@ -17,7 +17,6 @@ case class ListItem(
     source: String,
     direction: Direction,
     created: Timestamp,
-    fileCount: Int,
     corrOrg: Option[IdRef],
     corrPerson: Option[IdRef],
     concPerson: Option[IdRef],

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -43,6 +43,12 @@ object QItem {
   private val tag   = RTag.as("t")
   private val ti    = RTagItem.as("ti")
 
+  def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] =
+    Select(count(a.id).s, from(a), a.itemId.in(items)).build
+      .query[Int]
+      .unique
+      .map(_ + items.size)
+
   def findItem(id: Ident): ConnectionIO[Option[ItemData]] = {
     val ref = RItem.as("ref")
     val cq =
@@ -105,7 +111,6 @@ object QItem {
     ).build.query[ItemFieldValue].to[Vector]
 
   private def findItemsBase(q: Query.Fix, today: LocalDate, noteMaxLen: Int): Select = {
-    val attachs = AttachCountTable("cta")
     val coll = q.account.collective
 
     Select(
@@ -118,7 +123,6 @@ object QItem {
         i.source.s,
         i.incoming.s,
         i.created.s,
-        coalesce(attachs.num.s, const(0)).s,
         org.oid.s,
         org.name.s,
         pers0.pid.s,
@@ -136,17 +140,6 @@ object QItem {
       ),
       from(i)
         .leftJoin(f, f.id === i.folder && f.collective === coll)
-        .leftJoin(
-          Select(
-            select(countAll.as(attachs.num), a.itemId.as(attachs.itemId)),
-            from(a)
-              .innerJoin(i, i.id === a.itemId),
-            i.cid === q.account.collective,
-            GroupBy(a.itemId)
-          ),
-          attachs.aliasName,
-          attachs.itemId === i.id
-        )
         .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll)
         .leftJoin(org, org.oid === i.corrOrg && org.cid === coll)
         .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll)
@@ -158,7 +151,7 @@ object QItem {
             i.folder.in(QFolder.findMemberFolderIds(q.account))
           )
       )
-    ).distinct.orderBy(
+    ).orderBy(
       q.orderAsc
         .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
         .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
@@ -234,7 +227,7 @@ object QItem {
 
     val tagCloud =
       findItemsBase(q.fix, today, 0).unwrap
-        .withSelect(select(tag.all).append(count(i.id).as("num")))
+        .withSelect(select(tag.all).append(countDistinct(i.id).as("num")))
         .changeFrom(_.prepend(tagFrom))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
         .groupBy(tag.tid)
@@ -339,7 +332,7 @@ object QItem {
 
       object Tids extends TableDef {
         val tableName = "tids"
-        val alias: Option[String] = Some("tw")
+        val alias     = Some("tw")
         val itemId    = Column[Ident]("item_id", this)
         val weight    = Column[Double]("weight", this)
         val all       = Vector[Column[_]](itemId, weight)