Start converting QItem

modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala

@@ -121,7 +121,7 @@ object CreateItem {
 
   private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
     Task { ctx =>
-      val states      = ItemState.invalidStates.toList.toSet
+      val states      = ItemState.invalidStates
       val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet
       for {
         cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states))

modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala

@@ -105,7 +105,7 @@ object ItemHandler {
 
   private def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, Args, Unit] =
     Task { ctx =>
-      val states = ItemState.invalidStates.toList.toSet
+      val states = ItemState.invalidStates
       for {
         items <- ctx.store.transact(
           QItem.findByFileIds(ctx.args.files.map(_.fileMetaId), states)

modules/store/src/main/scala/docspell/store/qb/CteBind.scala

@@ -1,9 +1,12 @@
 package docspell.store.qb
 
-case class CteBind(name: TableDef, select: Select) {}
+case class CteBind(name: TableDef, coldef: Vector[Column[_]], select: Select) {}
 
 object CteBind {
 
   def apply(t: (TableDef, Select)): CteBind =
-    CteBind(t._1, t._2)
+    CteBind(t._1, Vector.empty, t._2)
+
+  def apply(name: TableDef, col: Column[_], cols: Column[_]*)(select: Select): CteBind =
+    CteBind(name, cols.toVector.prepended(col), select)
 }

modules/store/src/main/scala/docspell/store/qb/DSL.scala

@@ -25,6 +25,13 @@ trait DSL extends DoobieMeta {
   def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl =
     DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector)
 
+  def withCte(
+      name: TableDef,
+      col: Column[_],
+      cols: Column[_]*
+  ): Select => DSL.WithCteDsl =
+    sel => DSL.WithCteDsl(CteBind(name, col, cols: _*)(sel), Vector.empty)
+
   def select(cond: Condition): Nel[SelectExpr] =
     Nel.of(SelectExpr.SelectCondition(cond, None))
 

modules/store/src/main/scala/docspell/store/qb/Select.scala

@@ -21,9 +21,21 @@ sealed trait Select {
 
   def limit(n: Int): Select =
     this match {
-      case Select.Limit(q, _) => Select.Limit(q, n)
+      case Select.Limit(q, _) =>
-      case _                  => Select.Limit(this, n)
+        Select.Limit(q, n)
+      case _ =>
+        Select.Limit(this, n)
     }
+
+  def appendCte(next: CteBind): Select =
+    this match {
+      case Select.WithCte(cte, ctes, query) =>
+        Select.WithCte(cte, ctes :+ next, query)
+      case _ =>
+        Select.WithCte(next, Vector.empty, this)
+    }
+
+  def appendSelect(e: SelectExpr): Select
 }
 
 object Select {
@@ -69,16 +81,34 @@ object Select {
       copy(where = c)
     def where(c: Condition): SimpleSelect =
       copy(where = Some(c))
+
+    def appendSelect(e: SelectExpr): SimpleSelect =
+      copy(projection = projection.append(e))
   }
 
-  case class Union(q: Select, qs: Vector[Select]) extends Select
+  case class Union(q: Select, qs: Vector[Select]) extends Select {
+    def appendSelect(e: SelectExpr): Union =
+      copy(q = q.appendSelect(e))
+  }
 
-  case class Intersect(q: Select, qs: Vector[Select]) extends Select
+  case class Intersect(q: Select, qs: Vector[Select]) extends Select {
+    def appendSelect(e: SelectExpr): Intersect =
+      copy(q = q.appendSelect(e))
+  }
 
   case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy])
-      extends Select
+      extends Select {
+    def appendSelect(e: SelectExpr): Ordered =
+      copy(q = q.appendSelect(e))
+  }
 
-  case class Limit(q: Select, limit: Int) extends Select
+  case class Limit(q: Select, limit: Int) extends Select {
+    def appendSelect(e: SelectExpr): Limit =
+      copy(q = q.appendSelect(e))
+  }
 
-  case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select
+  case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select {
+    def appendSelect(e: SelectExpr): WithCte =
+      copy(query = query.appendSelect(e))
+  }
 }

modules/store/src/main/scala/docspell/store/qb/TableDef.scala

@@ -8,7 +8,7 @@ trait TableDef {
 
 object TableDef {
 
-  def apply(table: String, aliasName: Option[String] = None): TableDef =
+  def apply(table: String, aliasName: Option[String] = None): BasicTable =
     BasicTable(table, aliasName)
 
   final case class BasicTable(tableName: String, alias: Option[String]) extends TableDef {

modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala

@@ -18,3 +18,4 @@ trait CommonBuilder {
   def appendAs(alias: Option[String]): Fragment =
     alias.map(a => fr" AS" ++ Fragment.const(a)).getOrElse(Fragment.empty)
 }
+object CommonBuilder extends CommonBuilder

modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala

@@ -24,10 +24,10 @@ object DBFunctionBuilder extends CommonBuilder {
 
       case DBFunction.Coalesce(expr, exprs) =>
         val v = exprs.prepended(expr).map(SelectExprBuilder.build)
-        sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ sql")"
+        sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ fr")"
 
       case DBFunction.Power(expr, base) =>
-        sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")"
+        sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")"
 
       case DBFunction.Calc(op, left, right) =>
         SelectExprBuilder.build(left) ++

modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala

@@ -1,9 +1,9 @@
 package docspell.store.qb.impl
 
 import docspell.store.qb._
-
 import _root_.doobie.implicits._
 import _root_.doobie.{Query => _, _}
+import cats.data.NonEmptyList
 
 object SelectBuilder {
   val comma     = fr","
@@ -74,5 +74,19 @@ object SelectBuilder {
   }
 
   def buildCte(bind: CteBind): Fragment =
-    Fragment.const(bind.name.tableName) ++ sql"AS (" ++ build(bind.select) ++ sql")"
+    bind match {
+      case CteBind(name, cols, select) =>
+        val colDef =
+          NonEmptyList
+            .fromFoldable(cols)
+            .map(nel =>
+              nel
+                .map(col => CommonBuilder.columnNoPrefix(col))
+                .reduceLeft(_ ++ comma ++ _)
+            )
+            .map(f => sql"(" ++ f ++ sql")")
+            .getOrElse(Fragment.empty)
+
+        Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")"
+    }
 }

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -6,7 +6,6 @@ import cats.effect.Sync
 import cats.effect.concurrent.Ref
 import cats.implicits._
 import fs2.Stream
-
 import docspell.common.syntax.all._
 import docspell.common.{IdRef, _}
 import docspell.store.Store
@@ -14,7 +13,6 @@ import docspell.store.impl.Implicits._
 import docspell.store.impl._
 import docspell.store.qb.Select
 import docspell.store.records._
-
 import bitpeace.FileMeta
 import doobie._
 import doobie.implicits._
@@ -583,19 +581,18 @@ object QItem {
   }
 
   private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = {
-    val aId    = RAttachment.Columns.id.prefix("a")
+    import docspell.store.qb._
-    val aItem  = RAttachment.Columns.itemId.prefix("a")
+    import docspell.store.qb.DSL._
-    val aPos   = RAttachment.Columns.position.prefix("a")
-    val aName  = RAttachment.Columns.name.prefix("a")
-    val mId    = RAttachmentMeta.Columns.id.prefix("m")
-    val mPages = RAttachmentMeta.Columns.pages.prefix("m")
 
-    val cols = Seq(aId, aPos, aName, mPages)
+    val a = RAttachment.as("a")
-    val join = RAttachment.table ++
+    val m = RAttachmentMeta.as("m")
-      fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId)
-    val cond = aItem.is(item)
 
-    selectSimple(cols, join, cond).query[AttachmentLight].to[List]
+    Select(
+      select(a.id, a.position, a.name, m.pages),
+      from(a)
+        .leftJoin(m, m.id === a.id),
+      a.itemId === item
+    ).build.query[AttachmentLight].to[List]
   }
 
   def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
@@ -609,108 +606,73 @@ object QItem {
 
   private def findByFileIdsQuery(
       fileMetaIds: NonEmptyList[Ident],
-      limit: Option[Int],
+      states: Option[NonEmptyList[ItemState]]
-      states: Set[ItemState]
+  ): Select.SimpleSelect = {
-  ): Fragment = {
+    import docspell.store.qb._
-    val IC            = RItem.Columns.all.map(_.prefix("i"))
+    import docspell.store.qb.DSL._
-    val aItem         = RAttachment.Columns.itemId.prefix("a")
-    val aId           = RAttachment.Columns.id.prefix("a")
-    val aFileId       = RAttachment.Columns.fileId.prefix("a")
-    val iId           = RItem.Columns.id.prefix("i")
-    val iState        = RItem.Columns.state.prefix("i")
-    val sId           = RAttachmentSource.Columns.id.prefix("s")
-    val sFileId       = RAttachmentSource.Columns.fileId.prefix("s")
-    val rId           = RAttachmentArchive.Columns.id.prefix("r")
-    val rFileId       = RAttachmentArchive.Columns.fileId.prefix("r")
-    val m1            = RFileMeta.as("m1")
-    val m2            = RFileMeta.as("m2")
-    val m3            = RFileMeta.as("m3")
-    val m1Id          = m1.id.column
-    val m2Id          = m2.id.column
-    val m3Id          = m3.id.column
-    val filemetaTable = Fragment.const(RFileMeta.T.tableName)
 
-    val from =
+    val i = RItem.as("i")
-      RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++
+    val a = RAttachment.as("a")
-        fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++
+    val s = RAttachmentSource.as("s")
-        fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++
+    val r = RAttachmentArchive.as("r")
-        fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++
-        fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++
-        fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId)
 
-    val fileCond =
+    Select(
-      or(m1Id.isIn(fileMetaIds), m2Id.isIn(fileMetaIds), m3Id.isIn(fileMetaIds))
+      select(i.all),
-    val cond = NonEmptyList.fromList(states.toList) match {
+      from(i)
-      case Some(nel) =>
+        .innerJoin(a, a.itemId === i.id)
-        and(fileCond, iState.isIn(nel))
+        .innerJoin(s, s.id === a.id)
-      case None =>
+        .leftJoin(r, r.id === a.id),
-        fileCond
+      (a.fileId.in(fileMetaIds) ||
-    }
+        s.fileId.in(fileMetaIds) ||
-    val q = selectSimple(IC, from, cond)
+        r.fileId.in(fileMetaIds)) &&? states.map(nel => i.state.in(nel))
-
+    )
-    limit match {
-      case Some(n) => q ++ fr"LIMIT $n"
-      case None    => q
-    }
   }
 
   def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] =
     NonEmptyList.fromList(fileMetaIds.toList) match {
       case Some(nel) =>
-        findByFileIdsQuery(nel, Some(1), Set.empty).query[RItem].option
+        findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option
       case None =>
         (None: Option[RItem]).pure[ConnectionIO]
     }
 
   def findByFileIds(
       fileMetaIds: Seq[Ident],
-      states: Set[ItemState]
+      states: NonEmptyList[ItemState]
   ): ConnectionIO[Vector[RItem]] =
     NonEmptyList.fromList(fileMetaIds.toList) match {
       case Some(nel) =>
-        findByFileIdsQuery(nel, None, states).query[RItem].to[Vector]
+        findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector]
       case None =>
         Vector.empty[RItem].pure[ConnectionIO]
     }
 
   def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = {
-    val IC            = RItem.Columns.all.map(_.prefix("i"))
+    import docspell.store.qb._
-    val aItem         = RAttachment.Columns.itemId.prefix("a")
+    import docspell.store.qb.DSL._
-    val aId           = RAttachment.Columns.id.prefix("a")
-    val aFileId       = RAttachment.Columns.fileId.prefix("a")
-    val iId           = RItem.Columns.id.prefix("i")
-    val iColl         = RItem.Columns.cid.prefix("i")
-    val sId           = RAttachmentSource.Columns.id.prefix("s")
-    val sFileId       = RAttachmentSource.Columns.fileId.prefix("s")
-    val rId           = RAttachmentArchive.Columns.id.prefix("r")
-    val rFileId       = RAttachmentArchive.Columns.fileId.prefix("r")
-    val m1            = RFileMeta.as("m1")
-    val m2            = RFileMeta.as("m2")
-    val m3            = RFileMeta.as("m3")
-    val m1Id          = m1.id.column
-    val m2Id          = m2.id.column
-    val m3Id          = m3.id.column
-    val m1Checksum    = m1.checksum.column
-    val m2Checksum    = m2.checksum.column
-    val m3Checksum    = m3.checksum.column
-    val filemetaTable = Fragment.const(RFileMeta.T.tableName)
-    val from =
-      RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++
-        fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++
-        fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++
-        fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++
-        fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++
-        fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId)
 
-    selectSimple(
+    val m1 = RFileMeta.as("m1")
-      IC,
+    val m2 = RFileMeta.as("m2")
-      from,
+    val m3 = RFileMeta.as("m3")
-      and(
+    val i  = RItem.as("i")
-        or(m1Checksum.is(checksum), m2Checksum.is(checksum), m3Checksum.is(checksum)),
+    val a  = RAttachment.as("a")
-        iColl.is(collective)
+    val s  = RAttachmentSource.as("s")
+    val r  = RAttachmentArchive.as("r")
+
+    Select(
+      select(i.all),
+      from(i)
+        .innerJoin(a, a.itemId === i.id)
+        .innerJoin(s, s.id === a.id)
+        .innerJoin(m1, m1.id === a.fileId)
+        .innerJoin(m2, m2.id === s.fileId)
+        .leftJoin(r, r.id === a.id)
+        .leftJoin(m3, m3.id === r.fileId),
+      where(
+        i.cid === collective &&
+          (m1.checksum === checksum || m2.checksum === checksum || m3.checksum === checksum)
       )
-    ).query[RItem]
+    ).build.query[RItem].to[Vector]
-      .to[Vector]
   }
 
   final case class NameAndNotes(
@@ -724,15 +686,16 @@ object QItem {
       coll: Option[Ident],
       chunkSize: Int
   ): Stream[ConnectionIO, NameAndNotes] = {
-    val iId     = RItem.Columns.id
+    import docspell.store.qb._
-    val iColl   = RItem.Columns.cid
+    import docspell.store.qb.DSL._
-    val iName   = RItem.Columns.name
-    val iFolder = RItem.Columns.folder
-    val iNotes  = RItem.Columns.notes
 
-    val cols  = Seq(iId, iColl, iFolder, iName, iNotes)
+    val i = RItem.as("i")
-    val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty)
+
-    selectSimple(cols, RItem.table, where)
+    Select(
+      select(i.id, i.cid, i.folder, i.name, i.notes),
+      from(i)
+    ).where(coll.map(cid => i.cid === cid))
+      .build
       .query[NameAndNotes]
       .streamWithChunkSize(chunkSize)
   }
@@ -741,15 +704,13 @@ object QItem {
       collective: Ident,
       chunkSize: Int
   ): Stream[ConnectionIO, Ident] = {
-    val cols   = Seq(RItem.Columns.id)
+    import docspell.store.qb._
-    val iColl  = RItem.Columns.cid
+    import docspell.store.qb.DSL._
-    val iState = RItem.Columns.state
+
-    (selectSimple(
+    val i = RItem.as("i")
-      cols,
+    Select(i.id.s, from(i), i.cid === collective && i.state === ItemState.confirmed)
-      RItem.table,
+      .orderBy(i.created.desc)
-      and(iColl.is(collective), iState.is(ItemState.confirmed))
+      .build
-    ) ++
-      orderBy(RItem.Columns.created.desc))
       .query[Ident]
       .streamWithChunkSize(chunkSize)
   }
@@ -763,45 +724,39 @@ object QItem {
       tagCategory: String,
       pageSep: String
   ): ConnectionIO[TextAndTag] = {
-    val aId     = RAttachment.Columns.id.prefix("a")
+    import docspell.store.qb._
-    val aItem   = RAttachment.Columns.itemId.prefix("a")
+    import docspell.store.qb.DSL._
-    val mId     = RAttachmentMeta.Columns.id.prefix("m")
+
-    val mText   = RAttachmentMeta.Columns.content.prefix("m")
-    val tagItem = RTagItem.as("ti") //Columns.itemId.prefix("ti")
-    //val tiTag  = RTagItem.Columns.tagId.prefix("ti")
     val tag = RTag.as("t")
-//    val tId   = RTag.Columns.tid.prefix("t")
+    val a   = RAttachment.as("a")
-//    val tName = RTag.Columns.name.prefix("t")
+    val am  = RAttachmentMeta.as("m")
-//    val tCat  = RTag.Columns.category.prefix("t")
+    val ti  = RTagItem.as("ti")
-    val iId   = RItem.Columns.id.prefix("i")
+    val i   = RItem.as("i")
-    val iColl = RItem.Columns.cid.prefix("i")
 
-    val cte = withCTE(
+    val tags     = TableDef("tags").as("tt")
-      "tags" -> selectSimple(
+    val tagsItem = Column[Ident]("itemid", tags)
-        Seq(tagItem.itemId.column, tag.tid.column, tag.name.column),
+    val tagsTid  = Column[Ident]("tid", tags)
-        Fragment.const(RTagItem.t.tableName) ++ fr"ti INNER JOIN" ++
+    val tagsName = Column[String]("tname", tags)
-          Fragment.const(tag.tableName) ++ fr"t ON" ++ tag.tid.column
-            .is(tagItem.tagId.column),
-        and(tagItem.itemId.column.is(itemId), tag.category.column.is(tagCategory))
-      )
-    )
 
-    val cols = Seq(mText, tag.tid.column, tag.name.column)
+    val q =
+      withCte(
+        tags -> Select(
+          select(ti.itemId.as(tagsItem), tag.tid.as(tagsTid), tag.name.as(tagsName)),
+          from(ti)
+            .innerJoin(tag, tag.tid === ti.tagId),
+          ti.itemId === itemId && tag.category === tagCategory
+        )
+      )(
+        Select(
+          select(am.content, tagsTid, tagsName),
+          from(i)
+            .innerJoin(a, a.itemId === i.id)
+            .innerJoin(am, a.id === am.id)
+            .leftJoin(tags, tagsItem === i.id),
+          i.id === itemId && i.cid === collective && am.content.isNotNull && am.content <> ""
+        )
+      ).build
 
-    val from = RItem.table ++ fr"i INNER JOIN" ++
-      RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++
-      RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"LEFT JOIN" ++
-      fr"tags t ON" ++ RTagItem.t.itemId.oldColumn.prefix("t").is(iId)
-
-    val where =
-      and(
-        iId.is(itemId),
-        iColl.is(collective),
-        mText.isNotNull,
-        mText.isNot("")
-      )
-
-    val q = cte ++ selectDistinct(cols, from, where)
     for {
       _ <- logger.ftrace[ConnectionIO](
         s"query: $q  (${itemId.id}, ${collective.id}, ${tagCategory})"