Convert find items query

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -8,30 +8,30 @@ import docspell.backend.JobFactory
 import docspell.backend.ops.OItemSearch._
 import docspell.common._
 import docspell.ftsclient._
-import docspell.store.Store
 import docspell.store.queries.{QFolder, QItem}
 import docspell.store.queue.JobQueue
 import docspell.store.records.RJob
+import docspell.store.{Store, qb}
 
 trait OFulltext[F[_]] {
 
   def findItems(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
-      batch: Batch
+      batch: qb.Batch
   ): F[Vector[OFulltext.FtsItem]]
 
   /** Same as `findItems` but does more queries per item to find all tags. */
   def findItemsWithTags(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
-      batch: Batch
+      batch: qb.Batch
   ): F[Vector[OFulltext.FtsItemWithTags]]
 
   def findIndexOnly(maxNoteLen: Int)(
       fts: OFulltext.FtsInput,
       account: AccountId,
-      batch: Batch
+      batch: qb.Batch
   ): F[Vector[OFulltext.FtsItemWithTags]]
 
   /** Clears the full-text index completely and launches a task that
@@ -95,7 +95,7 @@ object OFulltext {
       def findIndexOnly(maxNoteLen: Int)(
           ftsQ: OFulltext.FtsInput,
           account: AccountId,
-          batch: Batch
+          batch: qb.Batch
       ): F[Vector[OFulltext.FtsItemWithTags]] = {
         val fq = FtsQuery(
           ftsQ.query,
@@ -135,7 +135,7 @@ object OFulltext {
 
       def findItems(
           maxNoteLen: Int
-      )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
+      )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] =
         findItemsFts(
           q,
           ftsQ,
@@ -152,7 +152,7 @@ object OFulltext {
       def findItemsWithTags(maxNoteLen: Int)(
           q: Query,
           ftsQ: FtsInput,
-          batch: Batch
+          batch: qb.Batch
       ): F[Vector[FtsItemWithTags]] =
         findItemsFts(
           q,
@@ -172,8 +172,8 @@ object OFulltext {
       private def findItemsFts[A: ItemId, B](
           q: Query,
           ftsQ: FtsInput,
-          batch: Batch,
-          search: (Query, Batch) => F[Vector[A]],
+          batch: qb.Batch,
+          search: (Query, qb.Batch) => F[Vector[A]],
           convert: (
               FtsResult,
               Map[Ident, List[FtsResult.ItemMatch]]
@@ -186,8 +186,8 @@ object OFulltext {
       private def findItemsFts0[A: ItemId, B](
           q: Query,
           ftsQ: FtsInput,
-          batch: Batch,
-          search: (Query, Batch) => F[Vector[A]],
+          batch: qb.Batch,
+          search: (Query, qb.Batch) => F[Vector[A]],
           convert: (
               FtsResult,
               Map[Ident, List[FtsResult.ItemMatch]]

modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala

@@ -7,9 +7,9 @@ import fs2.Stream
 
 import docspell.backend.ops.OItemSearch._
 import docspell.common._
-import docspell.store.Store
 import docspell.store.queries.{QAttachment, QItem}
 import docspell.store.records._
+import docspell.store.{Store, qb}
 
 import bitpeace.{FileMeta, RangeDef}
 import doobie.implicits._
@@ -59,8 +59,8 @@ object OItemSearch {
   type Query = QItem.Query
   val Query = QItem.Query
 
-  type Batch = docspell.store.queries.Batch
-  val Batch = docspell.store.queries.Batch
+  type Batch = qb.Batch
+  val Batch = docspell.store.qb.Batch
 
   type ListItem = QItem.ListItem
   val ListItem = QItem.ListItem

modules/store/src/main/scala/docspell/store/qb/Batch.scala

@@ -1,4 +1,4 @@
-package docspell.store.queries
+package docspell.store.qb
 
 case class Batch(offset: Int, limit: Int) {
   def restrictLimitTo(n: Int): Batch =

modules/store/src/main/scala/docspell/store/qb/Condition.scala

@@ -26,8 +26,36 @@ object Condition {
 
   case class IsNull(col: Column[_]) extends Condition
 
-  case class And(c: Condition, cs: Vector[Condition]) extends Condition
-  case class Or(c: Condition, cs: Vector[Condition])  extends Condition
-  case class Not(c: Condition)                        extends Condition
+  case class And(c: Condition, cs: Vector[Condition]) extends Condition {
+    def append(other: Condition): And =
+      other match {
+        case And(oc, ocs) =>
+          And(c, cs ++ (oc +: ocs))
+        case _ =>
+          And(c, cs :+ other)
+      }
+  }
+  object And {
+    def apply(c: Condition, cs: Condition*): And =
+      And(c, cs.toVector)
+
+  }
+
+  case class Or(c: Condition, cs: Vector[Condition]) extends Condition {
+    def append(other: Condition): Or =
+      other match {
+        case Or(oc, ocs) =>
+          Or(c, cs ++ (oc +: ocs))
+        case _ =>
+          Or(c, cs :+ other)
+      }
+  }
+  object Or {
+    def apply(c: Condition, cs: Condition*): Or =
+      Or(c, cs.toVector)
+  }
+
+  case class Not(c: Condition) extends Condition
+  object Not {}
 
 }

modules/store/src/main/scala/docspell/store/qb/DBFunction.scala

@@ -23,6 +23,8 @@ object DBFunction {
 
   case class Calc(op: Operator, left: SelectExpr, right: SelectExpr) extends DBFunction
 
+  case class Substring(expr: SelectExpr, start: Int, length: Int) extends DBFunction
+
   sealed trait Operator
   object Operator {
     case object Plus  extends Operator

modules/store/src/main/scala/docspell/store/qb/DSL.scala

@@ -80,6 +80,9 @@ trait DSL extends DoobieMeta {
   def power(base: Int, expr: SelectExpr): DBFunction =
     DBFunction.Power(expr, base)
 
+  def substring(expr: SelectExpr, start: Int, length: Int): DBFunction =
+    DBFunction.Substring(expr, start, length)
+
   def lit[A](value: A)(implicit P: Put[A]): SelectExpr.SelectLit[A] =
     SelectExpr.SelectLit(value, None)
 
@@ -91,8 +94,8 @@ trait DSL extends DoobieMeta {
 
   def and(c: Condition, cs: Condition*): Condition =
     c match {
-      case Condition.And(head, tail) =>
-        Condition.And(head, tail ++ (c +: cs.toVector))
+      case a: Condition.And =>
+        cs.foldLeft(a)(_.append(_))
       case _ =>
         Condition.And(c, cs.toVector)
     }
@@ -173,12 +176,21 @@ trait DSL extends DoobieMeta {
     def in(subsel: Select): Condition =
       Condition.InSubSelect(col, subsel)
 
+    def notIn(subsel: Select): Condition =
+      in(subsel).negate
+
     def in(values: Nel[A])(implicit P: Put[A]): Condition =
       Condition.InValues(col, values, false)
 
+    def notIn(values: Nel[A])(implicit P: Put[A]): Condition =
+      in(values).negate
+
     def inLower(values: Nel[A])(implicit P: Put[A]): Condition =
       Condition.InValues(col, values, true)
 
+    def notInLower(values: Nel[A])(implicit P: Put[A]): Condition =
+      Condition.InValues(col, values, true).negate
+
     def isNull: Condition =
       Condition.IsNull(col)
 
@@ -224,6 +236,9 @@ trait DSL extends DoobieMeta {
     def as(alias: String): SelectExpr =
       SelectExpr.SelectFun(dbf, Some(alias))
 
+    def as(otherCol: Column[_]): SelectExpr =
+      SelectExpr.SelectFun(dbf, Some(otherCol.name))
+
     def ===[A](value: A)(implicit P: Put[A]): Condition =
       Condition.CompareFVal(dbf, Operator.Eq, value)
 

modules/store/src/main/scala/docspell/store/qb/FromExpr.scala

@@ -1,6 +1,21 @@
 package docspell.store.qb
 
-sealed trait FromExpr
+import docspell.store.qb.FromExpr.{Joined, Relation}
+
+sealed trait FromExpr {
+  def innerJoin(other: Relation, on: Condition): Joined
+
+  def innerJoin(other: TableDef, on: Condition): Joined =
+    innerJoin(Relation.Table(other), on)
+
+  def leftJoin(other: Relation, on: Condition): Joined
+
+  def leftJoin(other: TableDef, on: Condition): Joined =
+    leftJoin(Relation.Table(other), on)
+
+  def leftJoin(sel: Select, alias: String, on: Condition): Joined =
+    leftJoin(Relation.SubSelect(sel, alias), on)
+}
 
 object FromExpr {
 
@@ -8,14 +23,8 @@ object FromExpr {
     def innerJoin(other: Relation, on: Condition): Joined =
       Joined(this, Vector(Join.InnerJoin(other, on)))
 
-    def innerJoin(other: TableDef, on: Condition): Joined =
-      innerJoin(Relation.Table(other), on)
-
     def leftJoin(other: Relation, on: Condition): Joined =
       Joined(this, Vector(Join.LeftJoin(other, on)))
-
-    def leftJoin(other: TableDef, on: Condition): Joined =
-      leftJoin(Relation.Table(other), on)
   }
 
   object From {
@@ -30,14 +39,9 @@ object FromExpr {
     def innerJoin(other: Relation, on: Condition): Joined =
       Joined(from, joins :+ Join.InnerJoin(other, on))
 
-    def innerJoin(other: TableDef, on: Condition): Joined =
-      innerJoin(Relation.Table(other), on)
-
     def leftJoin(other: Relation, on: Condition): Joined =
       Joined(from, joins :+ Join.LeftJoin(other, on))
 
-    def leftJoin(other: TableDef, on: Condition): Joined =
-      leftJoin(Relation.Table(other), on)
   }
 
   sealed trait Relation

modules/store/src/main/scala/docspell/store/qb/OrderBy.scala

@@ -6,6 +6,12 @@ final case class OrderBy(expr: SelectExpr, orderType: OrderType)
 
 object OrderBy {
 
+  def asc(e: SelectExpr): OrderBy =
+    OrderBy(e, OrderType.Asc)
+
+  def desc(e: SelectExpr): OrderBy =
+    OrderBy(e, OrderType.Desc)
+
   sealed trait OrderType
   object OrderType {
     case object Asc  extends OrderType

modules/store/src/main/scala/docspell/store/qb/Select.scala

@@ -13,20 +13,22 @@ sealed trait Select {
   def as(alias: String): SelectExpr.SelectQuery =
     SelectExpr.SelectQuery(this, Some(alias))
 
-  def orderBy(ob: OrderBy, obs: OrderBy*): Select.Ordered =
-    Select.Ordered(this, ob, obs.toVector)
+  def orderBy(ob: OrderBy, obs: OrderBy*): Select
 
-  def orderBy(c: Column[_]): Select.Ordered =
+  def orderBy(c: Column[_]): Select =
     orderBy(OrderBy(SelectExpr.SelectColumn(c, None), OrderBy.OrderType.Asc))
 
-  def limit(n: Int): Select =
+  def limit(batch: Batch): Select =
     this match {
       case Select.Limit(q, _) =>
-        Select.Limit(q, n)
+        Select.Limit(q, batch)
       case _ =>
-        Select.Limit(this, n)
+        Select.Limit(this, batch)
     }
 
+  def limit(n: Int): Select =
+    limit(Batch.limit(n))
+
   def appendCte(next: CteBind): Select =
     this match {
       case Select.WithCte(cte, ctes, query) =>
@@ -36,6 +38,10 @@ sealed trait Select {
     }
 
   def appendSelect(e: SelectExpr): Select
+
+  def changeFrom(f: FromExpr => FromExpr): Select
+
+  def changeWhere(f: Condition => Condition): Select
 }
 
 object Select {
@@ -84,36 +90,96 @@ object Select {
 
     def appendSelect(e: SelectExpr): SimpleSelect =
       copy(projection = projection.append(e))
+
+    def changeFrom(f: FromExpr => FromExpr): SimpleSelect =
+      copy(from = f(from))
+
+    def changeWhere(f: Condition => Condition): SimpleSelect =
+      copy(where = where.map(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Select =
+      Ordered(this, ob, obs.toVector)
   }
 
   case class RawSelect(fragment: Fragment) extends Select {
     def appendSelect(e: SelectExpr): RawSelect =
       sys.error("RawSelect doesn't support appending select expressions")
+
+    def changeFrom(f: FromExpr => FromExpr): Select =
+      sys.error("RawSelect doesn't support changing from expression")
+
+    def changeWhere(f: Condition => Condition): Select =
+      sys.error("RawSelect doesn't support changing where condition")
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Ordered =
+      sys.error("RawSelect doesn't support adding orderBy clause")
   }
 
   case class Union(q: Select, qs: Vector[Select]) extends Select {
     def appendSelect(e: SelectExpr): Union =
       copy(q = q.appendSelect(e))
+
+    def changeFrom(f: FromExpr => FromExpr): Union =
+      copy(q = q.changeFrom(f))
+
+    def changeWhere(f: Condition => Condition): Union =
+      copy(q = q.changeWhere(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Ordered =
+      Ordered(this, ob, obs.toVector)
   }
 
   case class Intersect(q: Select, qs: Vector[Select]) extends Select {
     def appendSelect(e: SelectExpr): Intersect =
       copy(q = q.appendSelect(e))
+
+    def changeFrom(f: FromExpr => FromExpr): Intersect =
+      copy(q = q.changeFrom(f))
+
+    def changeWhere(f: Condition => Condition): Intersect =
+      copy(q = q.changeWhere(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Ordered =
+      Ordered(this, ob, obs.toVector)
   }
 
   case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy])
       extends Select {
     def appendSelect(e: SelectExpr): Ordered =
       copy(q = q.appendSelect(e))
+    def changeFrom(f: FromExpr => FromExpr): Ordered =
+      copy(q = q.changeFrom(f))
+    def changeWhere(f: Condition => Condition): Ordered =
+      copy(q = q.changeWhere(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Ordered =
+      Ordered(q, ob, obs.toVector)
   }
 
-  case class Limit(q: Select, limit: Int) extends Select {
+  case class Limit(q: Select, batch: Batch) extends Select {
     def appendSelect(e: SelectExpr): Limit =
       copy(q = q.appendSelect(e))
+    def changeFrom(f: FromExpr => FromExpr): Limit =
+      copy(q = q.changeFrom(f))
+    def changeWhere(f: Condition => Condition): Limit =
+      copy(q = q.changeWhere(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): Limit =
+      copy(q = q.orderBy(ob, obs: _*))
+
   }
 
   case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select {
     def appendSelect(e: SelectExpr): WithCte =
       copy(query = query.appendSelect(e))
+
+    def changeFrom(f: FromExpr => FromExpr): WithCte =
+      copy(query = query.changeFrom(f))
+
+    def changeWhere(f: Condition => Condition): WithCte =
+      copy(query = query.changeWhere(f))
+
+    def orderBy(ob: OrderBy, obs: OrderBy*): WithCte =
+      copy(query = query.orderBy(ob, obs: _*))
   }
 }

modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala

@@ -29,6 +29,9 @@ object DBFunctionBuilder extends CommonBuilder {
       case DBFunction.Power(expr, base) =>
         sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")"
 
+      case DBFunction.Substring(expr, start, len) =>
+        sql"SUBSTRING(" ++ SelectExprBuilder.build(expr) ++ fr" FROM $start FOR $len)"
+
       case DBFunction.Calc(op, left, right) =>
         SelectExprBuilder.build(left) ++
           buildOperator(op) ++

modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala

@@ -1,9 +1,11 @@
 package docspell.store.qb.impl
 
+import cats.data.NonEmptyList
+
 import docspell.store.qb._
+
 import _root_.doobie.implicits._
 import _root_.doobie.{Query => _, _}
-import cats.data.NonEmptyList
 
 object SelectBuilder {
   val comma     = fr","
@@ -34,8 +36,8 @@ object SelectBuilder {
         val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _)
         build(q) ++ fr" ORDER BY" ++ order
 
-      case Select.Limit(q, n) =>
-        build(q) ++ fr" LIMIT $n"
+      case Select.Limit(q, batch) =>
+        build(q) ++ buildBatch(batch)
 
       case Select.WithCte(cte, moreCte, query) =>
         val ctes = moreCte.prepended(cte)
@@ -92,4 +94,16 @@ object SelectBuilder {
 
         Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")"
     }
+
+  def buildBatch(b: Batch): Fragment = {
+    val limitFrag =
+      if (b.limit != Int.MaxValue) fr" LIMIT ${b.limit}"
+      else Fragment.empty
+
+    val offsetFrag =
+      if (b.offset != 0) fr" OFFSET ${b.offset}"
+      else Fragment.empty
+
+    limitFrag ++ offsetFrag
+  }
 }

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -1,18 +1,19 @@
 package docspell.store.queries
 
-import cats.data.NonEmptyList
 import cats.data.OptionT
+import cats.data.{NonEmptyList => Nel}
 import cats.effect.Sync
 import cats.effect.concurrent.Ref
 import cats.implicits._
 import fs2.Stream
+
 import docspell.common.syntax.all._
 import docspell.common.{IdRef, _}
 import docspell.store.Store
-import docspell.store.impl.Implicits._
-import docspell.store.impl._
-import docspell.store.qb.Select
+import docspell.store.impl.DoobieMeta._
+import docspell.store.qb._
 import docspell.store.records._
+
 import bitpeace.FileMeta
 import doobie._
 import doobie.implicits._
@@ -86,6 +87,8 @@ object QItem {
   }
 
   def findItem(id: Ident): ConnectionIO[Option[ItemData]] = {
+    import docspell.store.impl.Implicits._
+
     val equip = REquipment.as("e")
     val org   = ROrganization.as("o")
     val pers0 = RPerson.as("p0")
@@ -226,7 +229,7 @@ object QItem {
       itemIds: Option[Set[Ident]],
       customValues: Seq[CustomValue],
       source: Option[String],
-      orderAsc: Option[RItem.Columns.type => Column]
+      orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]]
   )
 
   object Query {
@@ -260,7 +263,7 @@ object QItem {
   private def findCustomFieldValuesForColl(
       coll: Ident,
       values: Seq[CustomValue]
-  ): Seq[(String, Fragment)] = {
+  ): Option[Select] = {
     import docspell.store.qb.DSL._
 
     val cf = RCustomField.as("cf")
@@ -277,122 +280,90 @@ object QItem {
         )
       )
 
-    NonEmptyList.fromList(values.toList) match {
-      case Some(nel) =>
-        Seq("customvalues" -> intersect(nel.map(singleSelect)).build)
-      case None =>
-        Seq.empty
-    }
+    Nel
+      .fromList(values.toList)
+      .map(nel => intersect(nel.map(singleSelect)))
   }
 
-  private def findItemsBase(
-      q: Query,
-      distinct: Boolean,
-      noteMaxLen: Int,
-      moreCols: Seq[Fragment],
-      ctes: (String, Fragment)*
-  ): Fragment = {
+  private def findItemsBase(q: Query, noteMaxLen: Int): Select = {
+    import docspell.store.qb.DSL._
+
+    object Attachs extends TableDef {
+      val tableName = "attachs"
+      val aliasName = "cta"
+      val alias     = Some(aliasName)
+      val num       = Column[Int]("num", this)
+      val itemId    = Column[Ident]("item_id", this)
+    }
     val equip = REquipment.as("e1")
     val org   = ROrganization.as("o0")
-    val pers0 = RPerson.as("p0")
-    val pers1 = RPerson.as("p1")
+    val p0    = RPerson.as("p0")
+    val p1    = RPerson.as("p1")
     val f     = RFolder.as("f1")
-    val cv    = RCustomFieldValue.as("cv")
+    val i     = RItem.as("i")
+    val a     = RAttachment.as("a")
 
-    val IC         = RItem.Columns
-    val AC         = RAttachment.Columns
-    val itemCols   = IC.all
-    val equipCols  = List(equip.eid.oldColumn, equip.name.oldColumn)
-    val folderCols = List(f.id.oldColumn, f.name.oldColumn)
-    val cvItem     = cv.itemId.column
+    val coll = q.account.collective
 
-    val finalCols = commas(
-      Seq(
-        IC.id.prefix("i").f,
-        IC.name.prefix("i").f,
-        IC.state.prefix("i").f,
-        coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f),
-        IC.dueDate.prefix("i").f,
-        IC.source.prefix("i").f,
-        IC.incoming.prefix("i").f,
-        IC.created.prefix("i").f,
-        fr"COALESCE(a.num, 0)",
-        org.oid.column.f,
-        org.name.column.f,
-        pers0.pid.column.f,
-        pers0.name.column.f,
-        pers1.pid.column.f,
-        pers1.name.column.f,
-        equip.eid.oldColumn.prefix("e1").f,
-        equip.name.oldColumn.prefix("e1").f,
-        f.id.column.f,
-        f.name.column.f,
-        // sql uses 1 for first character
-        IC.notes.prefix("i").substring(1, noteMaxLen),
-        // last column is only for sorting
-        q.orderAsc match {
-          case Some(co) =>
-            coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
+    val baseSelect = Select(
+      select(
+        i.id.s,
+        i.name.s,
+        i.state.s,
+        coalesce(i.itemDate.s, i.created.s).s,
+        i.dueDate.s,
+        i.source.s,
+        i.incoming.s,
+        i.created.s,
+        coalesce(Attachs.num.s, lit(0)).s,
+        org.oid.s,
+        org.name.s,
+        p0.pid.s,
+        p0.name.s,
+        p1.pid.s,
+        p1.name.s,
+        equip.eid.s,
+        equip.name.s,
+        f.id.s,
+        f.name.s,
+        substring(i.notes.s, 1, noteMaxLen).s,
+        q.orderAsc
+          .map(of => coalesce(of(i).s, i.created.s).s)
+          .getOrElse(i.created.s)
+      ),
+      from(i)
+        .leftJoin(f, f.id === i.folder && f.collective === coll)
+        .leftJoin(
+          Select(
+            select(countAll.as(Attachs.num), a.itemId.as(Attachs.itemId)),
+            from(a)
+              .innerJoin(i, i.id === a.itemId),
+            i.cid === q.account.collective,
+            GroupBy(a.itemId)
+          ),
+          Attachs.aliasName, //alias, todo improve dsl
+          Attachs.itemId === i.id
+        )
+        .leftJoin(p0, p0.pid === i.corrPerson && p0.cid === coll)
+        .leftJoin(org, org.oid === i.corrOrg && org.cid === coll)
+        .leftJoin(p1, p1.pid === i.concPerson && p1.cid === coll)
+        .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll),
+      where(
+        i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&
+          or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account)))
+      )
+    ).distinct.orderBy(
+      q.orderAsc
+        .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
+        .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
+    )
+
+    findCustomFieldValuesForColl(coll, q.customValues) match {
+      case Some(itemIds) =>
+        baseSelect.changeWhere(c => c && i.id.in(itemIds))
       case None =>
-            IC.created.prefix("i").f
+        baseSelect
     }
-      ) ++ moreCols
-    )
-
-    val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.account.collective))
-    val withPerson =
-      selectSimple(
-        List(RPerson.T.pid.column, RPerson.T.name.column),
-        Fragment.const(RPerson.T.tableName),
-        RPerson.T.cid.column.is(q.account.collective)
-      )
-    val withOrgs =
-      selectSimple(
-        List(ROrganization.T.oid.column, ROrganization.T.name.column),
-        Fragment.const(ROrganization.T.tableName),
-        ROrganization.T.cid.column.is(q.account.collective)
-      )
-    val withEquips =
-      selectSimple(
-        equipCols,
-        Fragment.const(equip.tableName),
-        equip.cid.oldColumn.is(q.account.collective)
-      )
-    val withFolder =
-      selectSimple(
-        folderCols,
-        Fragment.const(f.tableName),
-        f.collective.oldColumn.is(q.account.collective)
-      )
-    val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++
-      fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")"
-
-    val withCustomValues =
-      findCustomFieldValuesForColl(q.account.collective, q.customValues)
-
-    val selectKW = if (distinct) fr"SELECT DISTINCT" else fr"SELECT"
-    withCTE(
-      (Seq(
-        "items"   -> withItem,
-        "persons" -> withPerson,
-        "orgs"    -> withOrgs,
-        "equips"  -> withEquips,
-        "attachs" -> withAttach,
-        "folders" -> withFolder
-      ) ++ withCustomValues ++ ctes): _*
-    ) ++
-      selectKW ++ finalCols ++ fr" FROM items i" ++
-      fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
-      fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(pers0.pid.column) ++
-      fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(org.oid.column) ++
-      fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(pers1.pid.column) ++
-      fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
-        .prefix("i")
-        .is(equip.eid.oldColumn.prefix("e1")) ++
-      fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(f.id.column) ++
-      (if (q.customValues.isEmpty) Fragment.empty
-       else
-         fr"INNER JOIN customvalues cv ON" ++ cvItem.is(IC.id.prefix("i")))
   }
 
   def findItems(
@@ -400,102 +371,54 @@ object QItem {
       maxNoteLen: Int,
       batch: Batch
   ): Stream[ConnectionIO, ListItem] = {
+    import docspell.store.qb.DSL._
+
     val equip = REquipment.as("e1")
     val org   = ROrganization.as("o0")
     val pers0 = RPerson.as("p0")
     val pers1 = RPerson.as("p1")
     val f     = RFolder.as("f1")
-    val IC    = RItem.Columns
+    val i     = RItem.as("i")
 
-    // inclusive tags are AND-ed
-    val tagSelectsIncl = q.tagsInclude
-      .map(tid =>
-        selectSimple(
-          List(RTagItem.t.itemId.column),
-          Fragment.const(RTagItem.t.tableName),
-          RTagItem.t.tagId.column.is(tid)
-        )
-      ) ++ q.tagCategoryIncl.map(cat => TagItemName.itemsInCategory(NonEmptyList.of(cat)))
-
-    // exclusive tags are OR-ed
-    val tagSelectsExcl =
-      TagItemName.itemsWithTagOrCategory(q.tagsExclude, q.tagCategoryExcl)
-
-    val iFolder    = IC.folder.prefix("i")
-    val name       = q.name.map(_.toLowerCase).map(QueryWildcard.apply)
-    val allNames   = q.allNames.map(_.toLowerCase).map(QueryWildcard.apply)
-    val sourceName = q.source.map(_.toLowerCase).map(QueryWildcard.apply)
-    val cond = and(
-      IC.cid.prefix("i").is(q.account.collective),
-      IC.state.prefix("i").isOneOf(q.states),
-      IC.incoming.prefix("i").isOrDiscard(q.direction),
-      name
-        .map(n => IC.name.prefix("i").lowerLike(n))
-        .getOrElse(Fragment.empty),
-      allNames
+    val cond: Condition => Condition =
+      c =>
+        c &&?
+          q.direction.map(d => i.incoming === d) &&?
+          q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&?
+          q.allNames
+            .map(QueryWildcard.lower)
             .map(n =>
-          or(
-            org.name.column.lowerLike(n),
-            pers0.name.column.lowerLike(n),
-            pers1.name.column.lowerLike(n),
-            equip.name.oldColumn.prefix("e1").lowerLike(n),
-            IC.name.prefix("i").lowerLike(n),
-            IC.notes.prefix("i").lowerLike(n)
-          )
-        )
-        .getOrElse(Fragment.empty),
-      pers0.pid.column.isOrDiscard(q.corrPerson),
-      org.oid.column.isOrDiscard(q.corrOrg),
-      pers1.pid.column.isOrDiscard(q.concPerson),
-      equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip),
-      f.id.column.isOrDiscard(q.folder),
-      if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty
-      else
-        IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl
-          .reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
-      if (q.tagsExclude.isEmpty && q.tagCategoryExcl.isEmpty) Fragment.empty
-      else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")",
-      q.dateFrom
-        .map(d =>
-          coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d"
-        )
-        .getOrElse(Fragment.empty),
-      q.dateTo
-        .map(d =>
-          coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d"
-        )
-        .getOrElse(Fragment.empty),
-      q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty),
-      q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty),
-      sourceName.map(s => IC.source.prefix("i").lowerLike(s)).getOrElse(Fragment.empty),
-      q.itemIds
-        .map(ids =>
-          NonEmptyList
-            .fromList(ids.toList)
-            .map(nel => IC.id.prefix("i").isIn(nel))
-            .getOrElse(IC.id.prefix("i").is(""))
-        )
-        .getOrElse(Fragment.empty),
-      or(iFolder.isNull, iFolder.isIn(QFolder.findMemberFolderIds(q.account).build))
-    )
+              org.name.like(n) ||
+                pers0.name.like(n) ||
+                pers1.name.like(n) ||
+                equip.name.like(n) ||
+                i.name.like(n) ||
+                i.notes.like(n)
+            ) &&?
+          q.corrPerson.map(p => pers0.pid === p) &&?
+          q.corrOrg.map(o => org.oid === o) &&?
+          q.concPerson.map(p => pers1.pid === p) &&?
+          q.concEquip.map(e => equip.eid === e) &&?
+          q.folder.map(fid => f.id === fid) &&?
+          q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&?
+          q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&?
+          q.dueDateFrom.map(d => i.dueDate > d) &&?
+          q.dueDateTo.map(d => i.dueDate < d) &&?
+          q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&?
+          q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&?
+          TagItemName
+            .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl)
+            .map(subsel => i.id.in(subsel)) &&?
+          TagItemName
+            .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl)
+            .map(subsel => i.id.notIn(subsel))
 
-    val order = q.orderAsc match {
-      case Some(co) =>
-        orderBy(coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) ++ fr"ASC")
-      case None =>
-        orderBy(
-          coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"DESC"
-        )
-    }
-    val limitOffset =
-      if (batch == Batch.all) Fragment.empty
-      else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
-
-    val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
-    val frag =
-      query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
-    logger.trace(s"List $batch items: $frag")
-    frag.query[ListItem].stream
+    val sql = findItemsBase(q, maxNoteLen)
+      .changeWhere(cond)
+      .limit(batch)
+      .build
+    logger.info(s"List $batch items: $sql")
+    sql.query[ListItem].stream
   }
 
   case class SelectedItem(itemId: Ident, weight: Double)
@@ -503,27 +426,50 @@ object QItem {
       q: Query,
       maxNoteLen: Int,
       items: Set[SelectedItem]
-  ): Stream[ConnectionIO, ListItem] =
+  ): Stream[ConnectionIO, ListItem] = {
+    import docspell.store.qb.DSL._
+
     if (items.isEmpty) Stream.empty
     else {
-      val IC = RItem.Columns
-      val values = items
+      val i = RItem.as("i")
+
+      object Tids extends TableDef {
+        val tableName             = "tids"
+        val alias: Option[String] = Some("tw")
+        val itemId                = Column[Ident]("item_id", this)
+        val weight                = Column[Double]("weight", this)
+        val all                   = Vector[Column[_]](itemId, weight)
+      }
+
+      val cte =
+        CteBind(
+          Tids,
+          Tids.all,
+          Select.RawSelect(
+            fr"VALUES" ++
+              items
                 .map(it => fr"(${it.itemId}, ${it.weight})")
                 .reduce((r, e) => r ++ fr"," ++ e)
+          )
+        )
 
-      val from = findItemsBase(
-        q,
-        true,
-        maxNoteLen,
-        Seq(fr"tids.weight"),
-        ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
-      ) ++
-        fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++
-        fr"ORDER BY tids.weight DESC"
+      val from = findItemsBase(q, maxNoteLen)
+        .appendCte(cte)
+        .appendSelect(Tids.weight.s)
+        .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id))
+        .orderBy(Tids.weight.desc)
+        .build
 
-      logger.trace(s"fts query: $from")
+//        Seq(fr"tids.weight"),
+//        ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
+//      ) ++
+//        fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++
+//        fr"ORDER BY tids.weight DESC"
+
+      logger.info(s"fts query: $from")
       from.query[ListItem].stream
     }
+  }
 
   case class AttachmentLight(
       id: Ident,
@@ -581,7 +527,6 @@ object QItem {
   }
 
   private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = {
-    import docspell.store.qb._
     import docspell.store.qb.DSL._
 
     val a = RAttachment.as("a")
@@ -605,10 +550,9 @@ object QItem {
     } yield tn + rn + n + mn + cf
 
   private def findByFileIdsQuery(
-      fileMetaIds: NonEmptyList[Ident],
-      states: Option[NonEmptyList[ItemState]]
+      fileMetaIds: Nel[Ident],
+      states: Option[Nel[ItemState]]
   ): Select.SimpleSelect = {
-    import docspell.store.qb._
     import docspell.store.qb.DSL._
 
     val i = RItem.as("i")
@@ -629,7 +573,7 @@ object QItem {
   }
 
   def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] =
-    NonEmptyList.fromList(fileMetaIds.toList) match {
+    Nel.fromList(fileMetaIds.toList) match {
       case Some(nel) =>
         findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option
       case None =>
@@ -638,9 +582,9 @@ object QItem {
 
   def findByFileIds(
       fileMetaIds: Seq[Ident],
-      states: NonEmptyList[ItemState]
+      states: Nel[ItemState]
   ): ConnectionIO[Vector[RItem]] =
-    NonEmptyList.fromList(fileMetaIds.toList) match {
+    Nel.fromList(fileMetaIds.toList) match {
       case Some(nel) =>
         findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector]
       case None =>
@@ -648,7 +592,6 @@ object QItem {
     }
 
   def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = {
-    import docspell.store.qb._
     import docspell.store.qb.DSL._
 
     val m1 = RFileMeta.as("m1")
@@ -704,7 +647,6 @@ object QItem {
       collective: Ident,
       chunkSize: Int
   ): Stream[ConnectionIO, Ident] = {
-    import docspell.store.qb._
     import docspell.store.qb.DSL._
 
     val i = RItem.as("i")
@@ -724,7 +666,6 @@ object QItem {
       tagCategory: String,
       pageSep: String
   ): ConnectionIO[TextAndTag] = {
-    import docspell.store.qb._
     import docspell.store.qb.DSL._
 
     val tag = RTag.as("t")

modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala

@@ -2,6 +2,9 @@ package docspell.store.queries
 
 object QueryWildcard {
 
+  def lower(s: String): String =
+    apply(s.toLowerCase)
+
   def apply(value: String): String = {
     def prefix(n: String) =
       if (n.startsWith("*")) s"%${n.substring(1)}"

modules/store/src/main/scala/docspell/store/records/RAttachment.scala

@@ -6,7 +6,6 @@ import fs2.Stream
 
 import docspell.common._
 import docspell.store.qb.DSL._
-import docspell.store.qb.TableDef
 import docspell.store.qb._
 
 import bitpeace.FileMeta
@@ -115,16 +114,6 @@ object RAttachment {
   def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = {
     import bitpeace.sql._
 
-//    val cols      = RFileMeta.Columns.all.map(_.prefix("m"))
-//    val aId       = id.prefix("a")
-//    val aFileMeta = fileId.prefix("a")
-//    val mId       = RFileMeta.Columns.id.prefix("m")
-//
-//    val from =
-//      table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId)
-//    val cond = aId.is(attachId)
-//
-//    selectSimple(cols, from, cond).query[FileMeta].option
     val m = RFileMeta.as("m")
     val a = RAttachment.as("a")
     Select(
@@ -167,14 +156,6 @@ object RAttachment {
       attachId: Ident,
       collective: Ident
   ): ConnectionIO[Boolean] = {
-//    val aId   = id.prefix("a")
-//    val aItem = itemId.prefix("a")
-//    val iId   = RItem.Columns.id.prefix("i")
-//    val iColl = RItem.Columns.cid.prefix("i")
-//    val from =
-//      table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId)
-//    val cond = and(iColl.is(collective), aId.is(attachId))
-//    selectCount(id, from, cond).query[Int].unique.map(_ > 0)
     val a = RAttachment.as("a")
     val i = RItem.as("i")
     Select(
@@ -189,12 +170,6 @@ object RAttachment {
       id: Ident,
       coll: Ident
   ): ConnectionIO[Vector[RAttachment]] = {
-//    val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++
-//      fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id
-//        .prefix("i")
-//        .is(itemId.prefix("a")) ++
-//      fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll))
-//    q.query[RAttachment].to[Vector]
     val a = RAttachment.as("a")
     val i = RItem.as("i")
     Select(
@@ -210,29 +185,6 @@ object RAttachment {
       coll: Ident,
       fileIds: NonEmptyList[Ident]
   ): ConnectionIO[Vector[RAttachment]] = {
-
-//    val iId   = RItem.Columns.id.prefix("i")
-//    val iColl = RItem.Columns.cid.prefix("i")
-//    val aItem = Columns.itemId.prefix("a")
-//    val aId   = Columns.id.prefix("a")
-//    val aFile = Columns.fileId.prefix("a")
-//    val sId   = RAttachmentSource.Columns.id.prefix("s")
-//    val sFile = RAttachmentSource.Columns.fileId.prefix("s")
-//    val rId   = RAttachmentArchive.Columns.id.prefix("r")
-//    val rFile = RAttachmentArchive.Columns.fileId.prefix("r")
-//
-//    val from = table ++ fr"a INNER JOIN" ++
-//      RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"LEFT JOIN" ++
-//      RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"LEFT JOIN" ++
-//      RAttachmentArchive.table ++ fr"r ON" ++ rId.is(aId)
-//
-//    val cond = and(
-//      iId.is(id),
-//      iColl.is(coll),
-//      or(aFile.isIn(fileIds), sFile.isIn(fileIds), rFile.isIn(fileIds))
-//    )
-//
-//    selectSimple(all.map(_.prefix("a")), from, cond).query[RAttachment].to[Vector]
     val i = RItem.as("i")
     val a = RAttachment.as("a")
     val s = RAttachmentSource.as("s")
@@ -255,19 +207,6 @@ object RAttachment {
   ): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
     import bitpeace.sql._
 
-//    val cols      = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m"))
-//    val afileMeta = fileId.prefix("a")
-//    val aItem     = itemId.prefix("a")
-//    val mId       = RFileMeta.Columns.id.prefix("m")
-//    val iId       = RItem.Columns.id.prefix("i")
-//    val iColl     = RItem.Columns.cid.prefix("i")
-//
-//    val from =
-//      table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++
-//        fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId)
-//    val cond = Seq(aItem.is(id), iColl.is(coll))
-//
-//    selectSimple(cols, from, and(cond)).query[(RAttachment, FileMeta)].to[Vector]
     val a = RAttachment.as("a")
     val m = RFileMeta.as("m")
     val i = RItem.as("i")
@@ -283,9 +222,6 @@ object RAttachment {
   def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
     import bitpeace.sql._
 
-//    val q =
-//      fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filzemeta m
-//      WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
     val a = RAttachment.as("a")
     val m = RFileMeta.as("m")
     Select(
@@ -313,24 +249,6 @@ object RAttachment {
       coll: Option[Ident],
       chunkSize: Int
   ): Stream[ConnectionIO, RAttachment] = {
-//    val aItem = Columns.itemId.prefix("a")
-//    val iId   = RItem.Columns.id.prefix("i")
-//    val iColl = RItem.Columns.cid.prefix("i")
-//
-//    val cols = all.map(_.prefix("a"))
-//
-//    coll match {
-//      case Some(cid) =>
-//        val join = table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem)
-//        val cond = iColl.is(cid)
-//        selectSimple(cols, join, cond)
-//          .query[RAttachment]
-//          .streamWithChunkSize(chunkSize)
-//      case None =>
-//        selectSimple(cols, table, Fragment.empty)
-//          .query[RAttachment]
-//          .streamWithChunkSize(chunkSize)
-//    }
     val a = RAttachment.as("a")
     val i = RItem.as("i")
 
@@ -350,19 +268,6 @@ object RAttachment {
   }
 
   def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = {
-//    val aId      = Columns.id.prefix("a")
-//    val aCreated = Columns.created.prefix("a")
-//    val mId      = RAttachmentMeta.Columns.id.prefix("m")
-//    val mPages   = RAttachmentMeta.Columns.pages.prefix("m")
-//
-//    val cols = all.map(_.prefix("a"))
-//    val join = table ++ fr"a LEFT OUTER JOIN" ++
-//      RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId)
-//    val cond = mPages.isNull
-//
-//    (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc))
-//      .query[RAttachment]
-//      .streamWithChunkSize(chunkSize)
     val a = RAttachment.as("a")
     val m = RAttachmentMeta.as("m")
     Select(
@@ -377,33 +282,6 @@ object RAttachment {
       coll: Option[Ident],
       chunkSize: Int
   ): Stream[ConnectionIO, RAttachment] = {
-//    val aId      = Columns.id.prefix("a")
-//    val aItem    = Columns.itemId.prefix("a")
-//    val aCreated = Columns.created.prefix("a")
-//    val pId      = RAttachmentPreview.Columns.id.prefix("p")
-//    val iId      = RItem.Columns.id.prefix("i")
-//    val iColl    = RItem.Columns.cid.prefix("i")
-//
-//    val cols = all.map(_.prefix("a"))
-//    val baseJoin =
-//      table ++ fr"a LEFT OUTER JOIN" ++
-//        RAttachmentPreview.table ++ fr"p ON" ++ pId.is(aId)
-//
-//    val baseCond =
-//      Seq(pId.isNull)
-//
-//    coll match {
-//      case Some(cid) =>
-//        val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem)
-//        val cond = and(baseCond ++ Seq(iColl.is(cid)))
-//        (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc))
-//          .query[RAttachment]
-//          .streamWithChunkSize(chunkSize)
-//      case None =>
-//        (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc))
-//          .query[RAttachment]
-//          .streamWithChunkSize(chunkSize)
-//    }
     val a = RAttachment.as("a")
     val p = RAttachmentPreview.as("p")
     val i = RItem.as("i")
@@ -420,28 +298,7 @@ object RAttachment {
       coll: Option[Ident],
       chunkSize: Int
   ): Stream[ConnectionIO, RAttachment] = {
-//    val aId     = Columns.id.prefix("a")
-//    val aItem   = Columns.itemId.prefix("a")
-//    val aFile   = Columns.fileId.prefix("a")
-//    val sId     = RAttachmentSource.Columns.id.prefix("s")
-//    val sFile   = RAttachmentSource.Columns.fileId.prefix("s")
-//    val iId     = RItem.Columns.id.prefix("i")
-//    val iColl   = RItem.Columns.cid.prefix("i")
-//    val mId     = RFileMeta.Columns.id.prefix("m")
-//    val mType   = RFileMeta.Columns.mimetype.prefix("m")
     val pdfType = "application/pdf%"
-//
-//    val from = table ++ fr"a INNER JOIN" ++
-//      RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"INNER JOIN" ++
-//      RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"INNER JOIN" ++
-//      RFileMeta.table ++ fr"m ON" ++ aFile.is(mId)
-//    val where = coll match {
-//      case Some(cid) => and(iColl.is(cid), aFile.is(sFile), mType.lowerLike(pdfType))
-//      case None      => and(aFile.is(sFile), mType.lowerLike(pdfType))
-//    }
-//    selectSimple(all.map(_.prefix("a")), from, where)
-//      .query[RAttachment]
-//      .streamWithChunkSize(chunkSize)
     val a       = RAttachment.as("a")
     val s       = RAttachmentSource.as("s")
     val i       = RItem.as("i")

modules/store/src/main/scala/docspell/store/records/RTagItem.scala

@@ -21,29 +21,29 @@ object RTagItem {
     val tagId     = Column[Ident]("tid", this)
     val all       = NonEmptyList.of[Column[_]](tagItemId, itemId, tagId)
   }
-  val t = Table(None)
+  val T = Table(None)
   def as(alias: String): Table =
     Table(Some(alias))
 
   def insert(v: RTagItem): ConnectionIO[Int] =
-    DML.insert(t, t.all, fr"${v.tagItemId},${v.itemId},${v.tagId}")
+    DML.insert(T, T.all, fr"${v.tagItemId},${v.itemId},${v.tagId}")
 
   def deleteItemTags(item: Ident): ConnectionIO[Int] =
-    DML.delete(t, t.itemId === item)
+    DML.delete(T, T.itemId === item)
 
   def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] =
-    DML.delete(t, t.itemId.in(RItem.filterItemsFragment(items, cid)))
+    DML.delete(T, T.itemId.in(RItem.filterItemsFragment(items, cid)))
 
   def deleteTag(tid: Ident): ConnectionIO[Int] =
-    DML.delete(t, t.tagId === tid)
+    DML.delete(T, T.tagId === tid)
 
   def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] =
-    run(select(t.all), from(t), t.itemId === item).query[RTagItem].to[Vector]
+    run(select(T.all), from(T), T.itemId === item).query[RTagItem].to[Vector]
 
   def findAllIn(item: Ident, tags: Seq[Ident]): ConnectionIO[Vector[RTagItem]] =
     NonEmptyList.fromList(tags.toList) match {
       case Some(nel) =>
-        run(select(t.all), from(t), t.itemId === item && t.tagId.in(nel))
+        run(select(T.all), from(T), T.itemId === item && T.tagId.in(nel))
           .query[RTagItem]
           .to[Vector]
       case None =>
@@ -55,7 +55,7 @@ object RTagItem {
       case None =>
         0.pure[ConnectionIO]
       case Some(nel) =>
-        DML.delete(t, t.itemId === item && t.tagId.in(nel))
+        DML.delete(T, T.itemId === item && T.tagId.in(nel))
     }
 
   def setAllTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
@@ -67,8 +67,8 @@ object RTagItem {
         )
         n <- DML
           .insertMany(
-            t,
-            t.all,
+            T,
+            T.all,
             entities.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
           )
       } yield n

modules/store/src/main/scala/docspell/store/records/TagItemName.scala

@@ -4,8 +4,7 @@ import cats.data.NonEmptyList
 
 import docspell.common._
 import docspell.store.qb.DSL._
-
-import doobie._
+import docspell.store.qb.{Condition, Select}
 
 /** A helper class combining information from `RTag` and `RTagItem`.
   * This is not a "record", there is no corresponding table.
@@ -20,32 +19,80 @@ case class TagItemName(
 )
 
 object TagItemName {
+  private val ti = RTagItem.as("ti")
+  private val t  = RTag.as("t")
 
-  def itemsInCategory(cats: NonEmptyList[String]): Fragment = {
-    val catsLower = cats.map(_.toLowerCase)
-    val ti        = RTagItem.as("ti")
-    val t         = RTag.as("t")
-    val join      = from(t).innerJoin(ti, t.tid === ti.tagId)
-    if (cats.tail.isEmpty)
-      run(select(ti.itemId), join, t.category.likes(catsLower.head))
-    else
-      run(select(ti.itemId), join, t.category.inLower(cats))
-  }
+  private val taggedItems =
+    from(t).innerJoin(ti, t.tid === ti.tagId)
 
-  def itemsWithTagOrCategory(tags: List[Ident], cats: List[String]): Fragment = {
+  private def orTags(tags: NonEmptyList[Ident]): Condition =
+    ti.tagId.in(tags)
+
+  private def orCategory(cats: NonEmptyList[String]): Condition =
+    t.category.inLower(cats)
+
+  def itemsInEitherCategory(cats: NonEmptyList[String]): Select =
+    Select(ti.itemId.s, taggedItems, orCategory(cats)).distinct
+
+  def itemsInAllCategories(cats: NonEmptyList[String]): Select =
+    intersect(
+      cats.map(cat => Select(ti.itemId.s, taggedItems, t.category === cat).distinct)
+    )
+
+  def itemsWithEitherTag(tags: NonEmptyList[Ident]): Select =
+    Select(ti.itemId.s, from(ti), orTags(tags)).distinct
+
+  def itemsWithAllTags(tags: NonEmptyList[Ident]): Select =
+    intersect(tags.map(tid => Select(ti.itemId.s, from(ti), ti.tagId === tid).distinct))
+
+  def itemsWithEitherTagOrCategory(
+      tags: NonEmptyList[Ident],
+      cats: NonEmptyList[String]
+  ): Select =
+    Select(ti.itemId.s, taggedItems, orTags(tags) || orCategory(cats))
+
+  def itemsWithAllTagAndCategory(
+      tags: NonEmptyList[Ident],
+      cats: NonEmptyList[String]
+  ): Select =
+    Select(
+      ti.itemId.s,
+      from(ti),
+      ti.itemId.in(itemsWithAllTags(tags)) &&
+        ti.itemId.in(itemsInAllCategories(cats))
+    )
+
+  def itemsWithEitherTagOrCategory(
+      tags: List[Ident],
+      cats: List[String]
+  ): Option[Select] = {
     val catsLower = cats.map(_.toLowerCase)
-    val ti        = RTagItem.as("ti")
-    val t         = RTag.as("t")
-    val join      = from(t).innerJoin(ti, t.tid === ti.tagId)
     (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match {
       case (Some(tagNel), Some(catNel)) =>
-        run(select(ti.itemId), join, t.tid.in(tagNel) || t.category.inLower(catNel))
+        Some(itemsWithEitherTagOrCategory(tagNel, catNel))
       case (Some(tagNel), None) =>
-        run(select(ti.itemId), join, t.tid.in(tagNel))
+        Some(itemsWithEitherTag(tagNel))
       case (None, Some(catNel)) =>
-        run(select(ti.itemId), join, t.category.inLower(catNel))
+        Some(itemsInEitherCategory(catNel))
       case (None, None) =>
-        Fragment.empty
+        None
+    }
+  }
+
+  def itemsWithAllTagAndCategory(
+      tags: List[Ident],
+      cats: List[String]
+  ): Option[Select] = {
+    val catsLower = cats.map(_.toLowerCase)
+    (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match {
+      case (Some(tagNel), Some(catNel)) =>
+        Some(itemsWithAllTagAndCategory(tagNel, catNel))
+      case (Some(tagNel), None) =>
+        Some(itemsWithAllTags(tagNel))
+      case (None, Some(catNel)) =>
+        Some(itemsInAllCategories(catNel))
+      case (None, None) =>
+        None
     }
   }
 }

modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala

@@ -48,14 +48,14 @@ object QueryBuilderTest extends SimpleTestSuite {
             assertEquals(f, FromExpr.From(c))
             assertEquals(2, joins.size)
             joins.head match {
-              case Join.InnerJoin(tbl, cond) =>
+              case FromExpr.Join.InnerJoin(FromExpr.Relation.Table(tbl), cond) =>
                 assertEquals(tbl, owner)
                 assertEquals(cond, c.ownerId === owner.id)
               case _ =>
                 fail("Unexpected join result")
             }
             joins.tail.head match {
-              case Join.LeftJoin(tbl, cond) =>
+              case FromExpr.Join.LeftJoin(FromExpr.Relation.Table(tbl), cond) =>
                 assertEquals(tbl, lecturer)
                 assertEquals(cond, c.lecturerId === lecturer.id)
               case _ =>

modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala

@@ -0,0 +1,19 @@
+package docspell.store.qb.impl
+
+import minitest._
+import docspell.store.qb._
+import docspell.store.qb.DSL._
+import docspell.store.qb.model.{CourseRecord, PersonRecord}
+
+object DSLTest extends SimpleTestSuite {
+
+  val course = CourseRecord.as("c")
+  val person = PersonRecord.as("p")
+
+  test("and") {
+    val c = course.lessons > 4 && person.id === 3 && person.name.like("%a%")
+    val expect =
+      Condition.And(course.lessons > 4, person.id === 3, person.name.like("%a%"))
+    assertEquals(c, expect)
+  }
+}