Refactor search to separate between a base query and user query

The `findBase` is adding only strictly required conditions. Everything
else comes from the user.

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -164,7 +164,7 @@ object OFulltext {
             .flatMap(r => Stream.emits(r.results.map(_.itemId)))
             .compile
             .to(Set)
-          q = Query.empty(account).copy(itemIds = itemIds.some)
+          q = Query.empty(account).withCond(_.copy(itemIds = itemIds.some))
           res <- store.transact(QItem.searchStats(q))
         } yield res
       }
@@ -208,7 +208,7 @@ object OFulltext {
           search <- itemSearch.findItems(0)(q, Batch.all)
           fq = FtsQuery(
             ftsQ.query,
-            q.account.collective,
+            q.fix.account.collective,
             search.map(_.id).toSet,
             Set.empty,
             500,
@@ -220,7 +220,7 @@ object OFulltext {
             .flatMap(r => Stream.emits(r.results.map(_.itemId)))
             .compile
             .to(Set)
-          qnext = q.copy(itemIds = items.some)
+          qnext = q.withCond(_.copy(itemIds = items.some))
           res <- store.transact(QItem.searchStats(qnext))
         } yield res
 
@@ -253,7 +253,7 @@ object OFulltext {
         val sqlResult = search(q, batch)
         val fq = FtsQuery(
           ftsQ.query,
-          q.account.collective,
+          q.fix.account.collective,
           Set.empty,
           Set.empty,
           0,

modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala

@@ -138,7 +138,9 @@ object OItemSearch {
         val search = QItem.findItems(q, maxNoteLen: Int, batch)
         store
           .transact(
-            QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)
+            QItem
+              .findItemsWithTags(q.fix.account.collective, search)
+              .take(batch.limit.toLong)
           )
           .compile
           .toVector

modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala

@@ -72,13 +72,16 @@ object NotifyDueItemsTask {
       q =
         Query
           .empty(ctx.args.account)
-          .copy(
+          .withOrder(orderAsc = _.dueDate)
-            states = ItemState.validStates.toList,
+          .withCond(
-            tagsInclude = ctx.args.tagsInclude,
+            _.copy(
-            tagsExclude = ctx.args.tagsExclude,
+              states = ItemState.validStates.toList,
-            dueDateFrom = ctx.args.daysBack.map(back => now - Duration.days(back.toLong)),
+              tagsInclude = ctx.args.tagsInclude,
-            dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong)),
+              tagsExclude = ctx.args.tagsExclude,
-            orderAsc = Some(_.dueDate)
+              dueDateFrom =
+                ctx.args.daysBack.map(back => now - Duration.days(back.toLong)),
+              dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong))
+            )
           )
       res <-
         ctx.store

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -145,31 +145,32 @@ trait Conversions {
 
   def mkQuery(m: ItemSearch, account: AccountId): OItemSearch.Query =
     OItemSearch.Query(
-      account,
+      OItemSearch.Query.Fix(account, None),
-      m.name,
+      OItemSearch.Query.QueryCond(
-      if (m.inbox) Seq(ItemState.Created)
+        m.name,
-      else ItemState.validStates.toList,
+        if (m.inbox) Seq(ItemState.Created)
-      m.direction,
+        else ItemState.validStates.toList,
-      m.corrPerson,
+        m.direction,
-      m.corrOrg,
+        m.corrPerson,
-      m.concPerson,
+        m.corrOrg,
-      m.concEquip,
+        m.concPerson,
-      m.folder,
+        m.concEquip,
-      m.tagsInclude.map(Ident.unsafe),
+        m.folder,
-      m.tagsExclude.map(Ident.unsafe),
+        m.tagsInclude.map(Ident.unsafe),
-      m.tagCategoriesInclude,
+        m.tagsExclude.map(Ident.unsafe),
-      m.tagCategoriesExclude,
+        m.tagCategoriesInclude,
-      m.dateFrom,
+        m.tagCategoriesExclude,
-      m.dateUntil,
+        m.dateFrom,
-      m.dueDateFrom,
+        m.dateUntil,
-      m.dueDateUntil,
+        m.dueDateFrom,
-      m.allNames,
+        m.dueDateUntil,
-      m.itemSubset
+        m.allNames,
-        .map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet)
+        m.itemSubset
-        .filter(_.nonEmpty),
+          .map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet)
-      m.customValues.map(mkCustomValue),
+          .filter(_.nonEmpty),
-      m.source,
+        m.customValues.map(mkCustomValue),
-      None
+        m.source
+      )
     )
 
   def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue =
@@ -182,7 +183,7 @@ trait Conversions {
       ItemLightGroup(g._1, g._2.map(mkItemLight).toList)
 
     val gs =
-      groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
+      groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
     ItemLightList(gs)
   }
 

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -117,7 +117,7 @@ object QItem {
       .map(nel => intersect(nel.map(singleSelect)))
   }
 
-  private def findItemsBase(q: Query, noteMaxLen: Int): Select = {
+  private def findItemsBase(q: Query.Fix, noteMaxLen: Int): Select = {
     object Attachs extends TableDef {
       val tableName = "attachs"
       val aliasName = "cta"
@@ -128,7 +128,7 @@ object QItem {
 
     val coll = q.account.collective
 
-    val baseSelect = Select(
+    Select(
       select(
         i.id.s,
         i.name.s,
@@ -172,27 +172,23 @@ object QItem {
         .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll)
         .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll),
       where(
-        i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&
+        i.cid === coll && or(
-          or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account)))
+          i.folder.isNull,
+          i.folder.in(QFolder.findMemberFolderIds(q.account))
+        )
       )
     ).distinct.orderBy(
       q.orderAsc
         .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
         .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
     )
-
-    findCustomFieldValuesForColl(coll, q.customValues) match {
-      case Some(itemIds) =>
-        baseSelect.changeWhere(c => c && i.id.in(itemIds))
-      case None =>
-        baseSelect
-    }
   }
 
-  def queryCondition(q: Query): Condition =
+  def queryCondition(coll: Ident, q: Query.QueryCond): Condition =
     Condition.unit &&?
       q.direction.map(d => i.incoming === d) &&?
       q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&?
+      Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&?
       q.allNames
         .map(QueryWildcard.lower)
         .map(n =>
@@ -221,15 +217,17 @@ object QItem {
         .map(subsel => i.id.in(subsel)) &&?
       TagItemName
         .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl)
-        .map(subsel => i.id.notIn(subsel))
+        .map(subsel => i.id.notIn(subsel)) &&?
+      findCustomFieldValuesForColl(coll, q.customValues)
+        .map(itemIds => i.id.in(itemIds))
 
   def findItems(
       q: Query,
       maxNoteLen: Int,
       batch: Batch
   ): Stream[ConnectionIO, ListItem] = {
-    val sql = findItemsBase(q, maxNoteLen)
+    val sql = findItemsBase(q.fix, maxNoteLen)
-      .changeWhere(c => c && queryCondition(q))
+      .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
       .limit(batch)
       .build
     logger.trace(s"List $batch items: $sql")
@@ -251,10 +249,10 @@ object QItem {
         .innerJoin(i, i.id === ti.itemId)
 
     val tagCloud =
-      findItemsBase(q, 0).unwrap
+      findItemsBase(q.fix, 0).unwrap
         .withSelect(select(tag.all).append(count(i.id).as("num")))
         .changeFrom(_.prepend(tagFrom))
-        .changeWhere(c => c && queryCondition(q))
+        .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
         .groupBy(tag.tid)
         .build
         .query[TagCount]
@@ -264,24 +262,24 @@ object QItem {
     // are not included they are fetched separately
     for {
       existing <- tagCloud
-      other    <- RTag.findOthers(q.account.collective, existing.map(_.tag.tagId))
+      other    <- RTag.findOthers(q.fix.account.collective, existing.map(_.tag.tagId))
     } yield existing ++ other.map(TagCount(_, 0))
   }
 
   def searchCountSummary(q: Query): ConnectionIO[Int] =
-    findItemsBase(q, 0).unwrap
+    findItemsBase(q.fix, 0).unwrap
       .withSelect(Nel.of(count(i.id).as("num")))
-      .changeWhere(c => c && queryCondition(q))
+      .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
       .build
       .query[Int]
       .unique
 
   def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = {
     val fu = RUser.as("fu")
-    findItemsBase(q, 0).unwrap
+    findItemsBase(q.fix, 0).unwrap
       .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num")))
       .changeFrom(_.innerJoin(fu, fu.uid === f.owner))
-      .changeWhere(c => c && queryCondition(q))
+      .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
       .groupBy(f.id, f.name, f.owner, fu.login)
       .build
       .query[FolderCount]
@@ -295,9 +293,9 @@ object QItem {
         .innerJoin(i, i.id === cv.itemId)
 
     val base =
-      findItemsBase(q, 0).unwrap
+      findItemsBase(q.fix, 0).unwrap
         .changeFrom(_.prepend(fieldJoin))
-        .changeWhere(c => c && queryCondition(q))
+        .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
         .groupBy(GroupBy(cf.all))
 
     val basicFields = Nel.of(
@@ -374,7 +372,7 @@ object QItem {
           )
         )
 
-      val from = findItemsBase(q, maxNoteLen)
+      val from = findItemsBase(q.fix, maxNoteLen)
         .appendCte(cte)
         .appendSelect(Tids.weight.s)
         .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id))

modules/store/src/main/scala/docspell/store/queries/Query.scala

@@ -1,57 +1,70 @@
 package docspell.store.queries
 
 import docspell.common._
+import docspell.store.qb.Column
 import docspell.store.records.RItem
 
-case class Query(
+case class Query(fix: Query.Fix, cond: Query.QueryCond) {
-    account: AccountId,
+  def withCond(f: Query.QueryCond => Query.QueryCond): Query =
-    name: Option[String],
+    copy(cond = f(cond))
-    states: Seq[ItemState],
+
-    direction: Option[Direction],
+  def withOrder(orderAsc: RItem.Table => Column[_]): Query =
-    corrPerson: Option[Ident],
+    copy(fix = fix.copy(orderAsc = Some(orderAsc)))
-    corrOrg: Option[Ident],
+}
-    concPerson: Option[Ident],
-    concEquip: Option[Ident],
-    folder: Option[Ident],
-    tagsInclude: List[Ident],
-    tagsExclude: List[Ident],
-    tagCategoryIncl: List[String],
-    tagCategoryExcl: List[String],
-    dateFrom: Option[Timestamp],
-    dateTo: Option[Timestamp],
-    dueDateFrom: Option[Timestamp],
-    dueDateTo: Option[Timestamp],
-    allNames: Option[String],
-    itemIds: Option[Set[Ident]],
-    customValues: Seq[CustomValue],
-    source: Option[String],
-    orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]]
-)
 
 object Query {
+
+  case class Fix(account: AccountId, orderAsc: Option[RItem.Table => Column[_]])
+
+  case class QueryCond(
+      name: Option[String],
+      states: Seq[ItemState],
+      direction: Option[Direction],
+      corrPerson: Option[Ident],
+      corrOrg: Option[Ident],
+      concPerson: Option[Ident],
+      concEquip: Option[Ident],
+      folder: Option[Ident],
+      tagsInclude: List[Ident],
+      tagsExclude: List[Ident],
+      tagCategoryIncl: List[String],
+      tagCategoryExcl: List[String],
+      dateFrom: Option[Timestamp],
+      dateTo: Option[Timestamp],
+      dueDateFrom: Option[Timestamp],
+      dueDateTo: Option[Timestamp],
+      allNames: Option[String],
+      itemIds: Option[Set[Ident]],
+      customValues: Seq[CustomValue],
+      source: Option[String]
+  )
+  object QueryCond {
+    val empty =
+      QueryCond(
+        None,
+        Seq.empty,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        Nil,
+        Nil,
+        Nil,
+        Nil,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        Seq.empty,
+        None
+      )
+  }
+
   def empty(account: AccountId): Query =
-    Query(
+    Query(Fix(account, None), QueryCond.empty)
-      account,
+
-      None,
-      Seq.empty,
-      None,
-      None,
-      None,
-      None,
-      None,
-      None,
-      Nil,
-      Nil,
-      Nil,
-      Nil,
-      None,
-      None,
-      None,
-      None,
-      None,
-      None,
-      Seq.empty,
-      None,
-      None
-    )
 }

modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala

@@ -23,7 +23,7 @@ object ItemQueryGeneratorTest extends SimpleTestSuite {
     RAttachmentMeta.as("m")
   )
 
-  test("migration") {
+  test("basic test") {
     val q = ItemQueryParser
       .parseUnsafe("(& name:hello date>=2020-02-01 (| source=expense folder=test ))")
     val cond = ItemQueryGenerator(tables, Ident.unsafe("coll"))(q)