Merge pull request #1040 from eikek/query-improvements

Query improvements

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -290,12 +290,15 @@ object OFulltext {
         val qres =
           for {
             items <- sqlResult
-            ids = items.map(a => ItemId[A].itemId(a))
+            ids    = items.map(a => ItemId[A].itemId(a))
+            idsNel = NonEmptyList.fromFoldable(ids)
             // must find all index results involving the items.
             // Currently there is one result per item + one result per
             // attachment
-            limit = items.map(a => ItemId[A].fileCount(a)).sum + items.size
-            ftsQ  = fq.copy(items = ids.toSet, limit = limit)
+            limit <- idsNel
+              .map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds)))
+              .getOrElse(0.pure[F])
+            ftsQ = fq.copy(items = ids.toSet, limit = limit)
             ftsR <- fts.search(ftsQ)
             ftsItems = ftsR.results.groupBy(_.itemId)
             res      = items.collect(convert(ftsR, ftsItems))
@@ -320,22 +323,19 @@ object OFulltext {
 
   trait ItemId[A] {
     def itemId(a: A): Ident
-
-    def fileCount(a: A): Int
   }
   object ItemId {
     def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev
 
-    def from[A](f: A => Ident, g: A => Int): ItemId[A] =
+    def from[A](f: A => Ident): ItemId[A] =
       new ItemId[A] {
-        def itemId(a: A)    = f(a)
-        def fileCount(a: A) = g(a)
+        def itemId(a: A) = f(a)
       }
 
     implicit val listItemId: ItemId[ListItem] =
-      ItemId.from(_.id, _.fileCount)
+      ItemId.from(_.id)
 
     implicit val listItemWithTagsId: ItemId[ListItemWithTags] =
-      ItemId.from(_.item.id, _.item.fileCount)
+      ItemId.from(_.item.id)
   }
 }

modules/store/src/main/resources/db/migration/h2/V1.26.0__missing_indexes.sql

@@ -0,0 +1,3 @@
+CREATE INDEX "joblog_id_created_idx" ON "joblog"("jid", "created");
+-- H2 doesn't support coalesce in create index
+--CREATE INDEX "item_itemdate_created_idx" ON "item"(coalesce("itemdate", "created"));

modules/store/src/main/resources/db/migration/mariadb/V1.26.0__missing_indexes.sql

@@ -0,0 +1,3 @@
+CREATE INDEX `joblog_id_created_idx` ON `joblog`(`jid`, `created`);
+-- MariaDB doesn't support coalesce in index
+--CREATE INDEX `item_itemdate_created_idx` ON `item`(coalesce(`itemdate`, `created`));

modules/store/src/main/resources/db/migration/postgresql/V1.26.0__missing_indexes.sql

@@ -0,0 +1,2 @@
+CREATE INDEX "joblog_id_created_idx" ON "joblog"("jid", "created");
+CREATE INDEX "item_itemdate_created_idx" ON "item"(coalesce("itemdate", "created"));

modules/store/src/main/scala/docspell/store/queries/ListItem.scala

@@ -17,7 +17,6 @@ case class ListItem(
     source: String,
     direction: Direction,
     created: Timestamp,
-    fileCount: Int,
     corrOrg: Option[IdRef],
     corrPerson: Option[IdRef],
     concPerson: Option[IdRef],

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -43,6 +43,12 @@ object QItem {
   private val tag   = RTag.as("t")
   private val ti    = RTagItem.as("ti")
 
+  def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] =
+    Select(count(a.id).s, from(a), a.itemId.in(items)).build
+      .query[Int]
+      .unique
+      .map(_ + items.size)
+
   def findItem(id: Ident): ConnectionIO[Option[ItemData]] = {
     val ref = RItem.as("ref")
     val cq =
@@ -105,8 +111,7 @@ object QItem {
     ).build.query[ItemFieldValue].to[Vector]
 
   private def findItemsBase(q: Query.Fix, today: LocalDate, noteMaxLen: Int): Select = {
-    val attachs = AttachCountTable("cta")
-    val coll    = q.account.collective
+    val coll = q.account.collective
 
     Select(
       select(
@@ -118,7 +123,6 @@ object QItem {
         i.source.s,
         i.incoming.s,
         i.created.s,
-        coalesce(attachs.num.s, const(0)).s,
         org.oid.s,
         org.name.s,
         pers0.pid.s,
@@ -136,17 +140,6 @@ object QItem {
       ),
       from(i)
         .leftJoin(f, f.id === i.folder && f.collective === coll)
-        .leftJoin(
-          Select(
-            select(countAll.as(attachs.num), a.itemId.as(attachs.itemId)),
-            from(a)
-              .innerJoin(i, i.id === a.itemId),
-            i.cid === q.account.collective,
-            GroupBy(a.itemId)
-          ),
-          attachs.aliasName,
-          attachs.itemId === i.id
-        )
         .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll)
         .leftJoin(org, org.oid === i.corrOrg && org.cid === coll)
         .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll)
@@ -158,7 +151,7 @@ object QItem {
             i.folder.in(QFolder.findMemberFolderIds(q.account))
           )
       )
-    ).distinct.orderBy(
+    ).orderBy(
       q.orderAsc
         .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
         .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
@@ -234,7 +227,7 @@ object QItem {
 
     val tagCloud =
       findItemsBase(q.fix, today, 0).unwrap
-        .withSelect(select(tag.all).append(count(i.id).as("num")))
+        .withSelect(select(tag.all).append(countDistinct(i.id).as("num")))
         .changeFrom(_.prepend(tagFrom))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
         .groupBy(tag.tid)
@@ -338,11 +331,11 @@ object QItem {
       val i = RItem.as("i")
 
       object Tids extends TableDef {
-        val tableName             = "tids"
-        val alias: Option[String] = Some("tw")
-        val itemId                = Column[Ident]("item_id", this)
-        val weight                = Column[Double]("weight", this)
-        val all                   = Vector[Column[_]](itemId, weight)
+        val tableName = "tids"
+        val alias     = Some("tw")
+        val itemId    = Column[Ident]("item_id", this)
+        val weight    = Column[Double]("weight", this)
+        val all       = Vector[Column[_]](itemId, weight)
       }
 
       val cte =