From 35c62049f566dc2832d4697a79f031399c6e3646 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 22:56:19 +0100 Subject: [PATCH] Start converting QItem --- .../docspell/joex/process/CreateItem.scala | 2 +- .../docspell/joex/process/ItemHandler.scala | 2 +- .../scala/docspell/store/qb/CteBind.scala | 7 +- .../main/scala/docspell/store/qb/DSL.scala | 7 + .../main/scala/docspell/store/qb/Select.scala | 44 +++- .../scala/docspell/store/qb/TableDef.scala | 2 +- .../store/qb/impl/CommonBuilder.scala | 1 + .../store/qb/impl/DBFunctionBuilder.scala | 4 +- .../store/qb/impl/SelectBuilder.scala | 18 +- .../scala/docspell/store/queries/QItem.scala | 243 +++++++----------- 10 files changed, 170 insertions(+), 160 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 92d275fa..fe21203b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -121,7 +121,7 @@ object CreateItem { private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = Task { ctx => - val states = ItemState.invalidStates.toList.toSet + val states = ItemState.invalidStates val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet for { cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states)) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 757493d6..c211ce5b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -105,7 +105,7 @@ object ItemHandler { private def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, Args, Unit] = Task { ctx => - val states = ItemState.invalidStates.toList.toSet + val states = ItemState.invalidStates for { items <- ctx.store.transact( QItem.findByFileIds(ctx.args.files.map(_.fileMetaId), states) diff --git a/modules/store/src/main/scala/docspell/store/qb/CteBind.scala b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala index 0a22a056..16e5d436 100644 --- a/modules/store/src/main/scala/docspell/store/qb/CteBind.scala +++ b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala @@ -1,9 +1,12 @@ package docspell.store.qb -case class CteBind(name: TableDef, select: Select) {} +case class CteBind(name: TableDef, coldef: Vector[Column[_]], select: Select) {} object CteBind { def apply(t: (TableDef, Select)): CteBind = - CteBind(t._1, t._2) + CteBind(t._1, Vector.empty, t._2) + + def apply(name: TableDef, col: Column[_], cols: Column[_]*)(select: Select): CteBind = + CteBind(name, cols.toVector.prepended(col), select) } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index dbddbdc9..4a1aa116 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -25,6 +25,13 @@ trait DSL extends DoobieMeta { def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl = DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector) + def withCte( + name: TableDef, + col: Column[_], + cols: Column[_]* + ): Select => DSL.WithCteDsl = + sel => DSL.WithCteDsl(CteBind(name, col, cols: _*)(sel), Vector.empty) + def select(cond: Condition): Nel[SelectExpr] = Nel.of(SelectExpr.SelectCondition(cond, None)) diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index e219ee03..74bd8b6c 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -21,9 +21,21 @@ sealed trait Select { def limit(n: Int): Select = this match { - case Select.Limit(q, _) => Select.Limit(q, n) - case _ => Select.Limit(this, n) + case Select.Limit(q, _) => + Select.Limit(q, n) + case _ => + Select.Limit(this, n) } + + def appendCte(next: CteBind): Select = + this match { + case Select.WithCte(cte, ctes, query) => + Select.WithCte(cte, ctes :+ next, query) + case _ => + Select.WithCte(next, Vector.empty, this) + } + + def appendSelect(e: SelectExpr): Select } object Select { @@ -69,16 +81,34 @@ object Select { copy(where = c) def where(c: Condition): SimpleSelect = copy(where = Some(c)) + + def appendSelect(e: SelectExpr): SimpleSelect = + copy(projection = projection.append(e)) } - case class Union(q: Select, qs: Vector[Select]) extends Select + case class Union(q: Select, qs: Vector[Select]) extends Select { + def appendSelect(e: SelectExpr): Union = + copy(q = q.appendSelect(e)) + } - case class Intersect(q: Select, qs: Vector[Select]) extends Select + case class Intersect(q: Select, qs: Vector[Select]) extends Select { + def appendSelect(e: SelectExpr): Intersect = + copy(q = q.appendSelect(e)) + } case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) - extends Select + extends Select { + def appendSelect(e: SelectExpr): Ordered = + copy(q = q.appendSelect(e)) + } - case class Limit(q: Select, limit: Int) extends Select + case class Limit(q: Select, limit: Int) extends Select { + def appendSelect(e: SelectExpr): Limit = + copy(q = q.appendSelect(e)) + } - case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select + case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select { + def appendSelect(e: SelectExpr): WithCte = + copy(query = query.appendSelect(e)) + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala index 4ef6cfa4..e78d3ff4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala +++ b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala @@ -8,7 +8,7 @@ trait TableDef { object TableDef { - def apply(table: String, aliasName: Option[String] = None): TableDef = + def apply(table: String, aliasName: Option[String] = None): BasicTable = BasicTable(table, aliasName) final case class BasicTable(tableName: String, alias: Option[String]) extends TableDef { diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala index 8b79cfdf..e0418e60 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala @@ -18,3 +18,4 @@ trait CommonBuilder { def appendAs(alias: Option[String]): Fragment = alias.map(a => fr" AS" ++ Fragment.const(a)).getOrElse(Fragment.empty) } +object CommonBuilder extends CommonBuilder diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index 494ec66c..a805f2dc 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -24,10 +24,10 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Coalesce(expr, exprs) => val v = exprs.prepended(expr).map(SelectExprBuilder.build) - sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ sql")" + sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ fr")" case DBFunction.Power(expr, base) => - sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")" + sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")" case DBFunction.Calc(op, left, right) => SelectExprBuilder.build(left) ++ diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index 23ca286e..40234b96 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -1,9 +1,9 @@ package docspell.store.qb.impl import docspell.store.qb._ - import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} +import cats.data.NonEmptyList object SelectBuilder { val comma = fr"," @@ -74,5 +74,19 @@ object SelectBuilder { } def buildCte(bind: CteBind): Fragment = - Fragment.const(bind.name.tableName) ++ sql"AS (" ++ build(bind.select) ++ sql")" + bind match { + case CteBind(name, cols, select) => + val colDef = + NonEmptyList + .fromFoldable(cols) + .map(nel => + nel + .map(col => CommonBuilder.columnNoPrefix(col)) + .reduceLeft(_ ++ comma ++ _) + ) + .map(f => sql"(" ++ f ++ sql")") + .getOrElse(Fragment.empty) + + Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")" + } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 5c9ca443..db4aa531 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -6,7 +6,6 @@ import cats.effect.Sync import cats.effect.concurrent.Ref import cats.implicits._ import fs2.Stream - import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store @@ -14,7 +13,6 @@ import docspell.store.impl.Implicits._ import docspell.store.impl._ import docspell.store.qb.Select import docspell.store.records._ - import bitpeace.FileMeta import doobie._ import doobie.implicits._ @@ -583,19 +581,18 @@ object QItem { } private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val aPos = RAttachment.Columns.position.prefix("a") - val aName = RAttachment.Columns.name.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mPages = RAttachmentMeta.Columns.pages.prefix("m") + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val cols = Seq(aId, aPos, aName, mPages) - val join = RAttachment.table ++ - fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) - val cond = aItem.is(item) + val a = RAttachment.as("a") + val m = RAttachmentMeta.as("m") - selectSimple(cols, join, cond).query[AttachmentLight].to[List] + Select( + select(a.id, a.position, a.name, m.pages), + from(a) + .leftJoin(m, m.id === a.id), + a.itemId === item + ).build.query[AttachmentLight].to[List] } def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = @@ -609,108 +606,73 @@ object QItem { private def findByFileIdsQuery( fileMetaIds: NonEmptyList[Ident], - limit: Option[Int], - states: Set[ItemState] - ): Fragment = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iState = RItem.Columns.state.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1 = RFileMeta.as("m1") - val m2 = RFileMeta.as("m2") - val m3 = RFileMeta.as("m3") - val m1Id = m1.id.column - val m2Id = m2.id.column - val m3Id = m3.id.column - val filemetaTable = Fragment.const(RFileMeta.T.tableName) + states: Option[NonEmptyList[ItemState]] + ): Select.SimpleSelect = { + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val from = - RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ - fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ - fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) + val i = RItem.as("i") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val r = RAttachmentArchive.as("r") - val fileCond = - or(m1Id.isIn(fileMetaIds), m2Id.isIn(fileMetaIds), m3Id.isIn(fileMetaIds)) - val cond = NonEmptyList.fromList(states.toList) match { - case Some(nel) => - and(fileCond, iState.isIn(nel)) - case None => - fileCond - } - val q = selectSimple(IC, from, cond) - - limit match { - case Some(n) => q ++ fr"LIMIT $n" - case None => q - } + Select( + select(i.all), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(s, s.id === a.id) + .leftJoin(r, r.id === a.id), + (a.fileId.in(fileMetaIds) || + s.fileId.in(fileMetaIds) || + r.fileId.in(fileMetaIds)) &&? states.map(nel => i.state.in(nel)) + ) } def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] = NonEmptyList.fromList(fileMetaIds.toList) match { case Some(nel) => - findByFileIdsQuery(nel, Some(1), Set.empty).query[RItem].option + findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option case None => (None: Option[RItem]).pure[ConnectionIO] } def findByFileIds( fileMetaIds: Seq[Ident], - states: Set[ItemState] + states: NonEmptyList[ItemState] ): ConnectionIO[Vector[RItem]] = NonEmptyList.fromList(fileMetaIds.toList) match { case Some(nel) => - findByFileIdsQuery(nel, None, states).query[RItem].to[Vector] + findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector] case None => Vector.empty[RItem].pure[ConnectionIO] } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1 = RFileMeta.as("m1") - val m2 = RFileMeta.as("m2") - val m3 = RFileMeta.as("m3") - val m1Id = m1.id.column - val m2Id = m2.id.column - val m3Id = m3.id.column - val m1Checksum = m1.checksum.column - val m2Checksum = m2.checksum.column - val m3Checksum = m3.checksum.column - val filemetaTable = Fragment.const(RFileMeta.T.tableName) - val from = - RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ - fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ - fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) + import docspell.store.qb._ + import docspell.store.qb.DSL._ - selectSimple( - IC, - from, - and( - or(m1Checksum.is(checksum), m2Checksum.is(checksum), m3Checksum.is(checksum)), - iColl.is(collective) + val m1 = RFileMeta.as("m1") + val m2 = RFileMeta.as("m2") + val m3 = RFileMeta.as("m3") + val i = RItem.as("i") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val r = RAttachmentArchive.as("r") + + Select( + select(i.all), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(s, s.id === a.id) + .innerJoin(m1, m1.id === a.fileId) + .innerJoin(m2, m2.id === s.fileId) + .leftJoin(r, r.id === a.id) + .leftJoin(m3, m3.id === r.fileId), + where( + i.cid === collective && + (m1.checksum === checksum || m2.checksum === checksum || m3.checksum === checksum) ) - ).query[RItem] - .to[Vector] + ).build.query[RItem].to[Vector] } final case class NameAndNotes( @@ -724,15 +686,16 @@ object QItem { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, NameAndNotes] = { - val iId = RItem.Columns.id - val iColl = RItem.Columns.cid - val iName = RItem.Columns.name - val iFolder = RItem.Columns.folder - val iNotes = RItem.Columns.notes + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val cols = Seq(iId, iColl, iFolder, iName, iNotes) - val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty) - selectSimple(cols, RItem.table, where) + val i = RItem.as("i") + + Select( + select(i.id, i.cid, i.folder, i.name, i.notes), + from(i) + ).where(coll.map(cid => i.cid === cid)) + .build .query[NameAndNotes] .streamWithChunkSize(chunkSize) } @@ -741,15 +704,13 @@ object QItem { collective: Ident, chunkSize: Int ): Stream[ConnectionIO, Ident] = { - val cols = Seq(RItem.Columns.id) - val iColl = RItem.Columns.cid - val iState = RItem.Columns.state - (selectSimple( - cols, - RItem.table, - and(iColl.is(collective), iState.is(ItemState.confirmed)) - ) ++ - orderBy(RItem.Columns.created.desc)) + import docspell.store.qb._ + import docspell.store.qb.DSL._ + + val i = RItem.as("i") + Select(i.id.s, from(i), i.cid === collective && i.state === ItemState.confirmed) + .orderBy(i.created.desc) + .build .query[Ident] .streamWithChunkSize(chunkSize) } @@ -763,45 +724,39 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mText = RAttachmentMeta.Columns.content.prefix("m") - val tagItem = RTagItem.as("ti") //Columns.itemId.prefix("ti") - //val tiTag = RTagItem.Columns.tagId.prefix("ti") + import docspell.store.qb._ + import docspell.store.qb.DSL._ + val tag = RTag.as("t") -// val tId = RTag.Columns.tid.prefix("t") -// val tName = RTag.Columns.name.prefix("t") -// val tCat = RTag.Columns.category.prefix("t") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") + val a = RAttachment.as("a") + val am = RAttachmentMeta.as("m") + val ti = RTagItem.as("ti") + val i = RItem.as("i") - val cte = withCTE( - "tags" -> selectSimple( - Seq(tagItem.itemId.column, tag.tid.column, tag.name.column), - Fragment.const(RTagItem.t.tableName) ++ fr"ti INNER JOIN" ++ - Fragment.const(tag.tableName) ++ fr"t ON" ++ tag.tid.column - .is(tagItem.tagId.column), - and(tagItem.itemId.column.is(itemId), tag.category.column.is(tagCategory)) - ) - ) + val tags = TableDef("tags").as("tt") + val tagsItem = Column[Ident]("itemid", tags) + val tagsTid = Column[Ident]("tid", tags) + val tagsName = Column[String]("tname", tags) - val cols = Seq(mText, tag.tid.column, tag.name.column) + val q = + withCte( + tags -> Select( + select(ti.itemId.as(tagsItem), tag.tid.as(tagsTid), tag.name.as(tagsName)), + from(ti) + .innerJoin(tag, tag.tid === ti.tagId), + ti.itemId === itemId && tag.category === tagCategory + ) + )( + Select( + select(am.content, tagsTid, tagsName), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(am, a.id === am.id) + .leftJoin(tags, tagsItem === i.id), + i.id === itemId && i.cid === collective && am.content.isNotNull && am.content <> "" + ) + ).build - val from = RItem.table ++ fr"i INNER JOIN" ++ - RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++ - RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"LEFT JOIN" ++ - fr"tags t ON" ++ RTagItem.t.itemId.oldColumn.prefix("t").is(iId) - - val where = - and( - iId.is(itemId), - iColl.is(collective), - mText.isNotNull, - mText.isNot("") - ) - - val q = cte ++ selectDistinct(cols, from, where) for { _ <- logger.ftrace[ConnectionIO]( s"query: $q (${itemId.id}, ${collective.id}, ${tagCategory})"