From a5a8c2553bf6f6047d683afe3fc9c20b609aea08 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 28 Aug 2021 19:11:13 +0200 Subject: [PATCH 1/2] Add indexes to better support certain queries --- .../resources/db/migration/h2/V1.26.0__missing_indexes.sql | 3 +++ .../db/migration/mariadb/V1.26.0__missing_indexes.sql | 3 +++ .../db/migration/postgresql/V1.26.0__missing_indexes.sql | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 modules/store/src/main/resources/db/migration/h2/V1.26.0__missing_indexes.sql create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.26.0__missing_indexes.sql create mode 100644 modules/store/src/main/resources/db/migration/postgresql/V1.26.0__missing_indexes.sql diff --git a/modules/store/src/main/resources/db/migration/h2/V1.26.0__missing_indexes.sql b/modules/store/src/main/resources/db/migration/h2/V1.26.0__missing_indexes.sql new file mode 100644 index 00000000..4742ebc3 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.26.0__missing_indexes.sql @@ -0,0 +1,3 @@ +CREATE INDEX "joblog_id_created_idx" ON "joblog"("jid", "created"); +-- H2 doesn't support coalesce in create index +--CREATE INDEX "item_itemdate_created_idx" ON "item"(coalesce("itemdate", "created")); diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.26.0__missing_indexes.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.26.0__missing_indexes.sql new file mode 100644 index 00000000..32ca1e03 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.26.0__missing_indexes.sql @@ -0,0 +1,3 @@ +CREATE INDEX `joblog_id_created_idx` ON `joblog`(`jid`, `created`); +-- MariaDB doesn't support coalesce in index +--CREATE INDEX `item_itemdate_created_idx` ON `item`(coalesce(`itemdate`, `created`)); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.26.0__missing_indexes.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.26.0__missing_indexes.sql new file mode 100644 index 00000000..755f36b9 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.26.0__missing_indexes.sql @@ -0,0 +1,2 @@ +CREATE INDEX "joblog_id_created_idx" ON "joblog"("jid", "created"); +CREATE INDEX "item_itemdate_created_idx" ON "item"(coalesce("itemdate", "created")); From 733096f9797ed126a0db96915fe163ffe636ce02 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 29 Aug 2021 00:21:49 +0200 Subject: [PATCH 2/2] Improve querying item results The join to retrieve the attachment count per item turned out to be very expensive. And it is not used anymore for the result, only to support fulltext search. It is now removed from the query. The DISTINCT keyword is also removed, because it is not necessary and it is expensive. With the DISTINCT removed, a new index (provided in the previous commit) can now be used to avoid sorting items. --- .../docspell/backend/ops/OFulltext.scala | 20 +++++------ .../docspell/store/queries/ListItem.scala | 1 - .../scala/docspell/store/queries/QItem.scala | 35 ++++++++----------- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index c3f48c12..b2b3cdd2 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -290,12 +290,15 @@ object OFulltext { val qres = for { items <- sqlResult - ids = items.map(a => ItemId[A].itemId(a)) + ids = items.map(a => ItemId[A].itemId(a)) + idsNel = NonEmptyList.fromFoldable(ids) // must find all index results involving the items. // Currently there is one result per item + one result per // attachment - limit = items.map(a => ItemId[A].fileCount(a)).sum + items.size - ftsQ = fq.copy(items = ids.toSet, limit = limit) + limit <- idsNel + .map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds))) + .getOrElse(0.pure[F]) + ftsQ = fq.copy(items = ids.toSet, limit = limit) ftsR <- fts.search(ftsQ) ftsItems = ftsR.results.groupBy(_.itemId) res = items.collect(convert(ftsR, ftsItems)) @@ -320,22 +323,19 @@ object OFulltext { trait ItemId[A] { def itemId(a: A): Ident - - def fileCount(a: A): Int } object ItemId { def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev - def from[A](f: A => Ident, g: A => Int): ItemId[A] = + def from[A](f: A => Ident): ItemId[A] = new ItemId[A] { - def itemId(a: A) = f(a) - def fileCount(a: A) = g(a) + def itemId(a: A) = f(a) } implicit val listItemId: ItemId[ListItem] = - ItemId.from(_.id, _.fileCount) + ItemId.from(_.id) implicit val listItemWithTagsId: ItemId[ListItemWithTags] = - ItemId.from(_.item.id, _.item.fileCount) + ItemId.from(_.item.id) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala index fe0b0a02..5f053c99 100644 --- a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala @@ -17,7 +17,6 @@ case class ListItem( source: String, direction: Direction, created: Timestamp, - fileCount: Int, corrOrg: Option[IdRef], corrPerson: Option[IdRef], concPerson: Option[IdRef], diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 5b860961..38a71235 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -43,6 +43,12 @@ object QItem { private val tag = RTag.as("t") private val ti = RTagItem.as("ti") + def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] = + Select(count(a.id).s, from(a), a.itemId.in(items)).build + .query[Int] + .unique + .map(_ + items.size) + def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { val ref = RItem.as("ref") val cq = @@ -105,8 +111,7 @@ object QItem { ).build.query[ItemFieldValue].to[Vector] private def findItemsBase(q: Query.Fix, today: LocalDate, noteMaxLen: Int): Select = { - val attachs = AttachCountTable("cta") - val coll = q.account.collective + val coll = q.account.collective Select( select( @@ -118,7 +123,6 @@ object QItem { i.source.s, i.incoming.s, i.created.s, - coalesce(attachs.num.s, const(0)).s, org.oid.s, org.name.s, pers0.pid.s, @@ -136,17 +140,6 @@ object QItem { ), from(i) .leftJoin(f, f.id === i.folder && f.collective === coll) - .leftJoin( - Select( - select(countAll.as(attachs.num), a.itemId.as(attachs.itemId)), - from(a) - .innerJoin(i, i.id === a.itemId), - i.cid === q.account.collective, - GroupBy(a.itemId) - ), - attachs.aliasName, - attachs.itemId === i.id - ) .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll) .leftJoin(org, org.oid === i.corrOrg && org.cid === coll) .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll) @@ -158,7 +151,7 @@ object QItem { i.folder.in(QFolder.findMemberFolderIds(q.account)) ) ) - ).distinct.orderBy( + ).orderBy( q.orderAsc .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s)) .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s)) @@ -234,7 +227,7 @@ object QItem { val tagCloud = findItemsBase(q.fix, today, 0).unwrap - .withSelect(select(tag.all).append(count(i.id).as("num"))) + .withSelect(select(tag.all).append(countDistinct(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(tag.tid) @@ -338,11 +331,11 @@ object QItem { val i = RItem.as("i") object Tids extends TableDef { - val tableName = "tids" - val alias: Option[String] = Some("tw") - val itemId = Column[Ident]("item_id", this) - val weight = Column[Double]("weight", this) - val all = Vector[Column[_]](itemId, weight) + val tableName = "tids" + val alias = Some("tw") + val itemId = Column[Ident]("item_id", this) + val weight = Column[Double]("weight", this) + val all = Vector[Column[_]](itemId, weight) } val cte =