From 266fec9eb58db3b4fac8515d3dc61d6608f7335d Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 11:50:35 +0100 Subject: [PATCH] Convert find items query --- .../docspell/backend/ops/OFulltext.scala | 22 +- .../docspell/backend/ops/OItemSearch.scala | 6 +- .../store/{queries => qb}/Batch.scala | 2 +- .../scala/docspell/store/qb/Condition.scala | 34 +- .../scala/docspell/store/qb/DBFunction.scala | 2 + .../main/scala/docspell/store/qb/DSL.scala | 19 +- .../scala/docspell/store/qb/FromExpr.scala | 28 +- .../scala/docspell/store/qb/OrderBy.scala | 6 + .../main/scala/docspell/store/qb/Select.scala | 80 +++- .../store/qb/impl/DBFunctionBuilder.scala | 3 + .../store/qb/impl/SelectBuilder.scala | 24 +- .../scala/docspell/store/queries/QItem.scala | 395 ++++++++---------- .../store/queries/QueryWildcard.scala | 3 + .../docspell/store/records/RAttachment.scala | 151 +------ .../docspell/store/records/RTagItem.scala | 20 +- .../docspell/store/records/TagItemName.scala | 87 +++- .../docspell/store/qb/QueryBuilderTest.scala | 4 +- .../docspell/store/qb/impl/DSLTest.scala | 19 + 18 files changed, 455 insertions(+), 450 deletions(-) rename modules/store/src/main/scala/docspell/store/{queries => qb}/Batch.scala (92%) create mode 100644 modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 9bc3f4a6..f9efed22 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -8,30 +8,30 @@ import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.ftsclient._ -import docspell.store.Store import docspell.store.queries.{QFolder, QItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob +import docspell.store.{Store, qb} trait OFulltext[F[_]] { def findItems(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItem]] /** Same as `findItems` but does more queries per item to find all tags. */ def findItemsWithTags(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] def findIndexOnly(maxNoteLen: Int)( fts: OFulltext.FtsInput, account: AccountId, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] /** Clears the full-text index completely and launches a task that @@ -95,7 +95,7 @@ object OFulltext { def findIndexOnly(maxNoteLen: Int)( ftsQ: OFulltext.FtsInput, account: AccountId, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] = { val fq = FtsQuery( ftsQ.query, @@ -135,7 +135,7 @@ object OFulltext { def findItems( maxNoteLen: Int - )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] = + )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] = findItemsFts( q, ftsQ, @@ -152,7 +152,7 @@ object OFulltext { def findItemsWithTags(maxNoteLen: Int)( q: Query, ftsQ: FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[FtsItemWithTags]] = findItemsFts( q, @@ -172,8 +172,8 @@ object OFulltext { private def findItemsFts[A: ItemId, B]( q: Query, ftsQ: FtsInput, - batch: Batch, - search: (Query, Batch) => F[Vector[A]], + batch: qb.Batch, + search: (Query, qb.Batch) => F[Vector[A]], convert: ( FtsResult, Map[Ident, List[FtsResult.ItemMatch]] @@ -186,8 +186,8 @@ object OFulltext { private def findItemsFts0[A: ItemId, B]( q: Query, ftsQ: FtsInput, - batch: Batch, - search: (Query, Batch) => F[Vector[A]], + batch: qb.Batch, + search: (Query, qb.Batch) => F[Vector[A]], convert: ( FtsResult, Map[Ident, List[FtsResult.ItemMatch]] diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 25efda8d..94270e6f 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -7,9 +7,9 @@ import fs2.Stream import docspell.backend.ops.OItemSearch._ import docspell.common._ -import docspell.store.Store import docspell.store.queries.{QAttachment, QItem} import docspell.store.records._ +import docspell.store.{Store, qb} import bitpeace.{FileMeta, RangeDef} import doobie.implicits._ @@ -59,8 +59,8 @@ object OItemSearch { type Query = QItem.Query val Query = QItem.Query - type Batch = docspell.store.queries.Batch - val Batch = docspell.store.queries.Batch + type Batch = qb.Batch + val Batch = docspell.store.qb.Batch type ListItem = QItem.ListItem val ListItem = QItem.ListItem diff --git a/modules/store/src/main/scala/docspell/store/queries/Batch.scala b/modules/store/src/main/scala/docspell/store/qb/Batch.scala similarity index 92% rename from modules/store/src/main/scala/docspell/store/queries/Batch.scala rename to modules/store/src/main/scala/docspell/store/qb/Batch.scala index d88ec957..26ffa148 100644 --- a/modules/store/src/main/scala/docspell/store/queries/Batch.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Batch.scala @@ -1,4 +1,4 @@ -package docspell.store.queries +package docspell.store.qb case class Batch(offset: Int, limit: Int) { def restrictLimitTo(n: Int): Batch = diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index c89b7178..2a1f3097 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -26,8 +26,36 @@ object Condition { case class IsNull(col: Column[_]) extends Condition - case class And(c: Condition, cs: Vector[Condition]) extends Condition - case class Or(c: Condition, cs: Vector[Condition]) extends Condition - case class Not(c: Condition) extends Condition + case class And(c: Condition, cs: Vector[Condition]) extends Condition { + def append(other: Condition): And = + other match { + case And(oc, ocs) => + And(c, cs ++ (oc +: ocs)) + case _ => + And(c, cs :+ other) + } + } + object And { + def apply(c: Condition, cs: Condition*): And = + And(c, cs.toVector) + + } + + case class Or(c: Condition, cs: Vector[Condition]) extends Condition { + def append(other: Condition): Or = + other match { + case Or(oc, ocs) => + Or(c, cs ++ (oc +: ocs)) + case _ => + Or(c, cs :+ other) + } + } + object Or { + def apply(c: Condition, cs: Condition*): Or = + Or(c, cs.toVector) + } + + case class Not(c: Condition) extends Condition + object Not {} } diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 24de3520..86ad7b1a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -23,6 +23,8 @@ object DBFunction { case class Calc(op: Operator, left: SelectExpr, right: SelectExpr) extends DBFunction + case class Substring(expr: SelectExpr, start: Int, length: Int) extends DBFunction + sealed trait Operator object Operator { case object Plus extends Operator diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 45905d84..97c80a2a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -80,6 +80,9 @@ trait DSL extends DoobieMeta { def power(base: Int, expr: SelectExpr): DBFunction = DBFunction.Power(expr, base) + def substring(expr: SelectExpr, start: Int, length: Int): DBFunction = + DBFunction.Substring(expr, start, length) + def lit[A](value: A)(implicit P: Put[A]): SelectExpr.SelectLit[A] = SelectExpr.SelectLit(value, None) @@ -91,8 +94,8 @@ trait DSL extends DoobieMeta { def and(c: Condition, cs: Condition*): Condition = c match { - case Condition.And(head, tail) => - Condition.And(head, tail ++ (c +: cs.toVector)) + case a: Condition.And => + cs.foldLeft(a)(_.append(_)) case _ => Condition.And(c, cs.toVector) } @@ -173,12 +176,21 @@ trait DSL extends DoobieMeta { def in(subsel: Select): Condition = Condition.InSubSelect(col, subsel) + def notIn(subsel: Select): Condition = + in(subsel).negate + def in(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, false) + def notIn(values: Nel[A])(implicit P: Put[A]): Condition = + in(values).negate + def inLower(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, true) + def notInLower(values: Nel[A])(implicit P: Put[A]): Condition = + Condition.InValues(col, values, true).negate + def isNull: Condition = Condition.IsNull(col) @@ -224,6 +236,9 @@ trait DSL extends DoobieMeta { def as(alias: String): SelectExpr = SelectExpr.SelectFun(dbf, Some(alias)) + def as(otherCol: Column[_]): SelectExpr = + SelectExpr.SelectFun(dbf, Some(otherCol.name)) + def ===[A](value: A)(implicit P: Put[A]): Condition = Condition.CompareFVal(dbf, Operator.Eq, value) diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index ac32d791..72486323 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -1,6 +1,21 @@ package docspell.store.qb -sealed trait FromExpr +import docspell.store.qb.FromExpr.{Joined, Relation} + +sealed trait FromExpr { + def innerJoin(other: Relation, on: Condition): Joined + + def innerJoin(other: TableDef, on: Condition): Joined = + innerJoin(Relation.Table(other), on) + + def leftJoin(other: Relation, on: Condition): Joined + + def leftJoin(other: TableDef, on: Condition): Joined = + leftJoin(Relation.Table(other), on) + + def leftJoin(sel: Select, alias: String, on: Condition): Joined = + leftJoin(Relation.SubSelect(sel, alias), on) +} object FromExpr { @@ -8,14 +23,8 @@ object FromExpr { def innerJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.InnerJoin(other, on))) - def innerJoin(other: TableDef, on: Condition): Joined = - innerJoin(Relation.Table(other), on) - def leftJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.LeftJoin(other, on))) - - def leftJoin(other: TableDef, on: Condition): Joined = - leftJoin(Relation.Table(other), on) } object From { @@ -30,14 +39,9 @@ object FromExpr { def innerJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.InnerJoin(other, on)) - def innerJoin(other: TableDef, on: Condition): Joined = - innerJoin(Relation.Table(other), on) - def leftJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.LeftJoin(other, on)) - def leftJoin(other: TableDef, on: Condition): Joined = - leftJoin(Relation.Table(other), on) } sealed trait Relation diff --git a/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala index 24d42cd4..128d3b81 100644 --- a/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala +++ b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala @@ -6,6 +6,12 @@ final case class OrderBy(expr: SelectExpr, orderType: OrderType) object OrderBy { + def asc(e: SelectExpr): OrderBy = + OrderBy(e, OrderType.Asc) + + def desc(e: SelectExpr): OrderBy = + OrderBy(e, OrderType.Desc) + sealed trait OrderType object OrderType { case object Asc extends OrderType diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index f1135f97..155109a8 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -13,20 +13,22 @@ sealed trait Select { def as(alias: String): SelectExpr.SelectQuery = SelectExpr.SelectQuery(this, Some(alias)) - def orderBy(ob: OrderBy, obs: OrderBy*): Select.Ordered = - Select.Ordered(this, ob, obs.toVector) + def orderBy(ob: OrderBy, obs: OrderBy*): Select - def orderBy(c: Column[_]): Select.Ordered = + def orderBy(c: Column[_]): Select = orderBy(OrderBy(SelectExpr.SelectColumn(c, None), OrderBy.OrderType.Asc)) - def limit(n: Int): Select = + def limit(batch: Batch): Select = this match { case Select.Limit(q, _) => - Select.Limit(q, n) + Select.Limit(q, batch) case _ => - Select.Limit(this, n) + Select.Limit(this, batch) } + def limit(n: Int): Select = + limit(Batch.limit(n)) + def appendCte(next: CteBind): Select = this match { case Select.WithCte(cte, ctes, query) => @@ -36,6 +38,10 @@ sealed trait Select { } def appendSelect(e: SelectExpr): Select + + def changeFrom(f: FromExpr => FromExpr): Select + + def changeWhere(f: Condition => Condition): Select } object Select { @@ -84,36 +90,96 @@ object Select { def appendSelect(e: SelectExpr): SimpleSelect = copy(projection = projection.append(e)) + + def changeFrom(f: FromExpr => FromExpr): SimpleSelect = + copy(from = f(from)) + + def changeWhere(f: Condition => Condition): SimpleSelect = + copy(where = where.map(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Select = + Ordered(this, ob, obs.toVector) } case class RawSelect(fragment: Fragment) extends Select { def appendSelect(e: SelectExpr): RawSelect = sys.error("RawSelect doesn't support appending select expressions") + + def changeFrom(f: FromExpr => FromExpr): Select = + sys.error("RawSelect doesn't support changing from expression") + + def changeWhere(f: Condition => Condition): Select = + sys.error("RawSelect doesn't support changing where condition") + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + sys.error("RawSelect doesn't support adding orderBy clause") } case class Union(q: Select, qs: Vector[Select]) extends Select { def appendSelect(e: SelectExpr): Union = copy(q = q.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): Union = + copy(q = q.changeFrom(f)) + + def changeWhere(f: Condition => Condition): Union = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(this, ob, obs.toVector) } case class Intersect(q: Select, qs: Vector[Select]) extends Select { def appendSelect(e: SelectExpr): Intersect = copy(q = q.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): Intersect = + copy(q = q.changeFrom(f)) + + def changeWhere(f: Condition => Condition): Intersect = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(this, ob, obs.toVector) } case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) extends Select { def appendSelect(e: SelectExpr): Ordered = copy(q = q.appendSelect(e)) + def changeFrom(f: FromExpr => FromExpr): Ordered = + copy(q = q.changeFrom(f)) + def changeWhere(f: Condition => Condition): Ordered = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(q, ob, obs.toVector) } - case class Limit(q: Select, limit: Int) extends Select { + case class Limit(q: Select, batch: Batch) extends Select { def appendSelect(e: SelectExpr): Limit = copy(q = q.appendSelect(e)) + def changeFrom(f: FromExpr => FromExpr): Limit = + copy(q = q.changeFrom(f)) + def changeWhere(f: Condition => Condition): Limit = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Limit = + copy(q = q.orderBy(ob, obs: _*)) + } case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select { def appendSelect(e: SelectExpr): WithCte = copy(query = query.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): WithCte = + copy(query = query.changeFrom(f)) + + def changeWhere(f: Condition => Condition): WithCte = + copy(query = query.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): WithCte = + copy(query = query.orderBy(ob, obs: _*)) } } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index a805f2dc..d99302e8 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -29,6 +29,9 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Power(expr, base) => sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")" + case DBFunction.Substring(expr, start, len) => + sql"SUBSTRING(" ++ SelectExprBuilder.build(expr) ++ fr" FROM $start FOR $len)" + case DBFunction.Calc(op, left, right) => SelectExprBuilder.build(left) ++ buildOperator(op) ++ diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index ee50ba67..68743737 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -1,16 +1,18 @@ package docspell.store.qb.impl +import cats.data.NonEmptyList + import docspell.store.qb._ + import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} -import cats.data.NonEmptyList object SelectBuilder { val comma = fr"," val asc = fr" ASC" val desc = fr" DESC" - val intersect = fr"INTERSECT" - val union = fr"UNION ALL" + val intersect = fr" INTERSECT" + val union = fr" UNION ALL" def apply(q: Select): Fragment = build(q) @@ -34,8 +36,8 @@ object SelectBuilder { val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _) build(q) ++ fr" ORDER BY" ++ order - case Select.Limit(q, n) => - build(q) ++ fr" LIMIT $n" + case Select.Limit(q, batch) => + build(q) ++ buildBatch(batch) case Select.WithCte(cte, moreCte, query) => val ctes = moreCte.prepended(cte) @@ -92,4 +94,16 @@ object SelectBuilder { Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")" } + + def buildBatch(b: Batch): Fragment = { + val limitFrag = + if (b.limit != Int.MaxValue) fr" LIMIT ${b.limit}" + else Fragment.empty + + val offsetFrag = + if (b.offset != 0) fr" OFFSET ${b.offset}" + else Fragment.empty + + limitFrag ++ offsetFrag + } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index db4aa531..cea2177e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -1,18 +1,19 @@ package docspell.store.queries -import cats.data.NonEmptyList import cats.data.OptionT +import cats.data.{NonEmptyList => Nel} import cats.effect.Sync import cats.effect.concurrent.Ref import cats.implicits._ import fs2.Stream + import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store -import docspell.store.impl.Implicits._ -import docspell.store.impl._ -import docspell.store.qb.Select +import docspell.store.impl.DoobieMeta._ +import docspell.store.qb._ import docspell.store.records._ + import bitpeace.FileMeta import doobie._ import doobie.implicits._ @@ -86,6 +87,8 @@ object QItem { } def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { + import docspell.store.impl.Implicits._ + val equip = REquipment.as("e") val org = ROrganization.as("o") val pers0 = RPerson.as("p0") @@ -226,7 +229,7 @@ object QItem { itemIds: Option[Set[Ident]], customValues: Seq[CustomValue], source: Option[String], - orderAsc: Option[RItem.Columns.type => Column] + orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]] ) object Query { @@ -260,7 +263,7 @@ object QItem { private def findCustomFieldValuesForColl( coll: Ident, values: Seq[CustomValue] - ): Seq[(String, Fragment)] = { + ): Option[Select] = { import docspell.store.qb.DSL._ val cf = RCustomField.as("cf") @@ -277,122 +280,90 @@ object QItem { ) ) - NonEmptyList.fromList(values.toList) match { - case Some(nel) => - Seq("customvalues" -> intersect(nel.map(singleSelect)).build) - case None => - Seq.empty - } + Nel + .fromList(values.toList) + .map(nel => intersect(nel.map(singleSelect))) } - private def findItemsBase( - q: Query, - distinct: Boolean, - noteMaxLen: Int, - moreCols: Seq[Fragment], - ctes: (String, Fragment)* - ): Fragment = { + private def findItemsBase(q: Query, noteMaxLen: Int): Select = { + import docspell.store.qb.DSL._ + + object Attachs extends TableDef { + val tableName = "attachs" + val aliasName = "cta" + val alias = Some(aliasName) + val num = Column[Int]("num", this) + val itemId = Column[Ident]("item_id", this) + } val equip = REquipment.as("e1") val org = ROrganization.as("o0") - val pers0 = RPerson.as("p0") - val pers1 = RPerson.as("p1") + val p0 = RPerson.as("p0") + val p1 = RPerson.as("p1") val f = RFolder.as("f1") - val cv = RCustomFieldValue.as("cv") + val i = RItem.as("i") + val a = RAttachment.as("a") - val IC = RItem.Columns - val AC = RAttachment.Columns - val itemCols = IC.all - val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) - val folderCols = List(f.id.oldColumn, f.name.oldColumn) - val cvItem = cv.itemId.column + val coll = q.account.collective - val finalCols = commas( - Seq( - IC.id.prefix("i").f, - IC.name.prefix("i").f, - IC.state.prefix("i").f, - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f), - IC.dueDate.prefix("i").f, - IC.source.prefix("i").f, - IC.incoming.prefix("i").f, - IC.created.prefix("i").f, - fr"COALESCE(a.num, 0)", - org.oid.column.f, - org.name.column.f, - pers0.pid.column.f, - pers0.name.column.f, - pers1.pid.column.f, - pers1.name.column.f, - equip.eid.oldColumn.prefix("e1").f, - equip.name.oldColumn.prefix("e1").f, - f.id.column.f, - f.name.column.f, - // sql uses 1 for first character - IC.notes.prefix("i").substring(1, noteMaxLen), - // last column is only for sorting - q.orderAsc match { - case Some(co) => - coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) - case None => - IC.created.prefix("i").f - } - ) ++ moreCols + val baseSelect = Select( + select( + i.id.s, + i.name.s, + i.state.s, + coalesce(i.itemDate.s, i.created.s).s, + i.dueDate.s, + i.source.s, + i.incoming.s, + i.created.s, + coalesce(Attachs.num.s, lit(0)).s, + org.oid.s, + org.name.s, + p0.pid.s, + p0.name.s, + p1.pid.s, + p1.name.s, + equip.eid.s, + equip.name.s, + f.id.s, + f.name.s, + substring(i.notes.s, 1, noteMaxLen).s, + q.orderAsc + .map(of => coalesce(of(i).s, i.created.s).s) + .getOrElse(i.created.s) + ), + from(i) + .leftJoin(f, f.id === i.folder && f.collective === coll) + .leftJoin( + Select( + select(countAll.as(Attachs.num), a.itemId.as(Attachs.itemId)), + from(a) + .innerJoin(i, i.id === a.itemId), + i.cid === q.account.collective, + GroupBy(a.itemId) + ), + Attachs.aliasName, //alias, todo improve dsl + Attachs.itemId === i.id + ) + .leftJoin(p0, p0.pid === i.corrPerson && p0.cid === coll) + .leftJoin(org, org.oid === i.corrOrg && org.cid === coll) + .leftJoin(p1, p1.pid === i.concPerson && p1.cid === coll) + .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll), + where( + i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) && + or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account))) + ) + ).distinct.orderBy( + q.orderAsc + .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s)) + .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s)) ) - val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.account.collective)) - val withPerson = - selectSimple( - List(RPerson.T.pid.column, RPerson.T.name.column), - Fragment.const(RPerson.T.tableName), - RPerson.T.cid.column.is(q.account.collective) - ) - val withOrgs = - selectSimple( - List(ROrganization.T.oid.column, ROrganization.T.name.column), - Fragment.const(ROrganization.T.tableName), - ROrganization.T.cid.column.is(q.account.collective) - ) - val withEquips = - selectSimple( - equipCols, - Fragment.const(equip.tableName), - equip.cid.oldColumn.is(q.account.collective) - ) - val withFolder = - selectSimple( - folderCols, - Fragment.const(f.tableName), - f.collective.oldColumn.is(q.account.collective) - ) - val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ - fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" - - val withCustomValues = - findCustomFieldValuesForColl(q.account.collective, q.customValues) - - val selectKW = if (distinct) fr"SELECT DISTINCT" else fr"SELECT" - withCTE( - (Seq( - "items" -> withItem, - "persons" -> withPerson, - "orgs" -> withOrgs, - "equips" -> withEquips, - "attachs" -> withAttach, - "folders" -> withFolder - ) ++ withCustomValues ++ ctes): _* - ) ++ - selectKW ++ finalCols ++ fr" FROM items i" ++ - fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ - fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(pers0.pid.column) ++ - fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(org.oid.column) ++ - fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(pers1.pid.column) ++ - fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment - .prefix("i") - .is(equip.eid.oldColumn.prefix("e1")) ++ - fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(f.id.column) ++ - (if (q.customValues.isEmpty) Fragment.empty - else - fr"INNER JOIN customvalues cv ON" ++ cvItem.is(IC.id.prefix("i"))) + findCustomFieldValuesForColl(coll, q.customValues) match { + case Some(itemIds) => + baseSelect.changeWhere(c => c && i.id.in(itemIds)) + case None => + baseSelect + } } def findItems( @@ -400,102 +371,54 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { + import docspell.store.qb.DSL._ + val equip = REquipment.as("e1") val org = ROrganization.as("o0") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") val f = RFolder.as("f1") - val IC = RItem.Columns + val i = RItem.as("i") - // inclusive tags are AND-ed - val tagSelectsIncl = q.tagsInclude - .map(tid => - selectSimple( - List(RTagItem.t.itemId.column), - Fragment.const(RTagItem.t.tableName), - RTagItem.t.tagId.column.is(tid) - ) - ) ++ q.tagCategoryIncl.map(cat => TagItemName.itemsInCategory(NonEmptyList.of(cat))) + val cond: Condition => Condition = + c => + c &&? + q.direction.map(d => i.incoming === d) &&? + q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? + q.allNames + .map(QueryWildcard.lower) + .map(n => + org.name.like(n) || + pers0.name.like(n) || + pers1.name.like(n) || + equip.name.like(n) || + i.name.like(n) || + i.notes.like(n) + ) &&? + q.corrPerson.map(p => pers0.pid === p) &&? + q.corrOrg.map(o => org.oid === o) &&? + q.concPerson.map(p => pers1.pid === p) &&? + q.concEquip.map(e => equip.eid === e) &&? + q.folder.map(fid => f.id === fid) &&? + q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&? + q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&? + q.dueDateFrom.map(d => i.dueDate > d) &&? + q.dueDateTo.map(d => i.dueDate < d) &&? + q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? + q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&? + TagItemName + .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) + .map(subsel => i.id.in(subsel)) &&? + TagItemName + .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) + .map(subsel => i.id.notIn(subsel)) - // exclusive tags are OR-ed - val tagSelectsExcl = - TagItemName.itemsWithTagOrCategory(q.tagsExclude, q.tagCategoryExcl) - - val iFolder = IC.folder.prefix("i") - val name = q.name.map(_.toLowerCase).map(QueryWildcard.apply) - val allNames = q.allNames.map(_.toLowerCase).map(QueryWildcard.apply) - val sourceName = q.source.map(_.toLowerCase).map(QueryWildcard.apply) - val cond = and( - IC.cid.prefix("i").is(q.account.collective), - IC.state.prefix("i").isOneOf(q.states), - IC.incoming.prefix("i").isOrDiscard(q.direction), - name - .map(n => IC.name.prefix("i").lowerLike(n)) - .getOrElse(Fragment.empty), - allNames - .map(n => - or( - org.name.column.lowerLike(n), - pers0.name.column.lowerLike(n), - pers1.name.column.lowerLike(n), - equip.name.oldColumn.prefix("e1").lowerLike(n), - IC.name.prefix("i").lowerLike(n), - IC.notes.prefix("i").lowerLike(n) - ) - ) - .getOrElse(Fragment.empty), - pers0.pid.column.isOrDiscard(q.corrPerson), - org.oid.column.isOrDiscard(q.corrOrg), - pers1.pid.column.isOrDiscard(q.concPerson), - equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip), - f.id.column.isOrDiscard(q.folder), - if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty - else - IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl - .reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", - if (q.tagsExclude.isEmpty && q.tagCategoryExcl.isEmpty) Fragment.empty - else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")", - q.dateFrom - .map(d => - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d" - ) - .getOrElse(Fragment.empty), - q.dateTo - .map(d => - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d" - ) - .getOrElse(Fragment.empty), - q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty), - q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty), - sourceName.map(s => IC.source.prefix("i").lowerLike(s)).getOrElse(Fragment.empty), - q.itemIds - .map(ids => - NonEmptyList - .fromList(ids.toList) - .map(nel => IC.id.prefix("i").isIn(nel)) - .getOrElse(IC.id.prefix("i").is("")) - ) - .getOrElse(Fragment.empty), - or(iFolder.isNull, iFolder.isIn(QFolder.findMemberFolderIds(q.account).build)) - ) - - val order = q.orderAsc match { - case Some(co) => - orderBy(coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) ++ fr"ASC") - case None => - orderBy( - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"DESC" - ) - } - val limitOffset = - if (batch == Batch.all) Fragment.empty - else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}" - - val query = findItemsBase(q, true, maxNoteLen, Seq.empty) - val frag = - query ++ fr"WHERE" ++ cond ++ order ++ limitOffset - logger.trace(s"List $batch items: $frag") - frag.query[ListItem].stream + val sql = findItemsBase(q, maxNoteLen) + .changeWhere(cond) + .limit(batch) + .build + logger.info(s"List $batch items: $sql") + sql.query[ListItem].stream } case class SelectedItem(itemId: Ident, weight: Double) @@ -503,27 +426,50 @@ object QItem { q: Query, maxNoteLen: Int, items: Set[SelectedItem] - ): Stream[ConnectionIO, ListItem] = + ): Stream[ConnectionIO, ListItem] = { + import docspell.store.qb.DSL._ + if (items.isEmpty) Stream.empty else { - val IC = RItem.Columns - val values = items - .map(it => fr"(${it.itemId}, ${it.weight})") - .reduce((r, e) => r ++ fr"," ++ e) + val i = RItem.as("i") - val from = findItemsBase( - q, - true, - maxNoteLen, - Seq(fr"tids.weight"), - ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") - ) ++ - fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++ - fr"ORDER BY tids.weight DESC" + object Tids extends TableDef { + val tableName = "tids" + val alias: Option[String] = Some("tw") + val itemId = Column[Ident]("item_id", this) + val weight = Column[Double]("weight", this) + val all = Vector[Column[_]](itemId, weight) + } - logger.trace(s"fts query: $from") + val cte = + CteBind( + Tids, + Tids.all, + Select.RawSelect( + fr"VALUES" ++ + items + .map(it => fr"(${it.itemId}, ${it.weight})") + .reduce((r, e) => r ++ fr"," ++ e) + ) + ) + + val from = findItemsBase(q, maxNoteLen) + .appendCte(cte) + .appendSelect(Tids.weight.s) + .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id)) + .orderBy(Tids.weight.desc) + .build + +// Seq(fr"tids.weight"), +// ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") +// ) ++ +// fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++ +// fr"ORDER BY tids.weight DESC" + + logger.info(s"fts query: $from") from.query[ListItem].stream } + } case class AttachmentLight( id: Ident, @@ -581,7 +527,6 @@ object QItem { } private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val a = RAttachment.as("a") @@ -605,10 +550,9 @@ object QItem { } yield tn + rn + n + mn + cf private def findByFileIdsQuery( - fileMetaIds: NonEmptyList[Ident], - states: Option[NonEmptyList[ItemState]] + fileMetaIds: Nel[Ident], + states: Option[Nel[ItemState]] ): Select.SimpleSelect = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val i = RItem.as("i") @@ -629,7 +573,7 @@ object QItem { } def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] = - NonEmptyList.fromList(fileMetaIds.toList) match { + Nel.fromList(fileMetaIds.toList) match { case Some(nel) => findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option case None => @@ -638,9 +582,9 @@ object QItem { def findByFileIds( fileMetaIds: Seq[Ident], - states: NonEmptyList[ItemState] + states: Nel[ItemState] ): ConnectionIO[Vector[RItem]] = - NonEmptyList.fromList(fileMetaIds.toList) match { + Nel.fromList(fileMetaIds.toList) match { case Some(nel) => findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector] case None => @@ -648,7 +592,6 @@ object QItem { } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val m1 = RFileMeta.as("m1") @@ -704,7 +647,6 @@ object QItem { collective: Ident, chunkSize: Int ): Stream[ConnectionIO, Ident] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val i = RItem.as("i") @@ -724,7 +666,6 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val tag = RTag.as("t") diff --git a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala index 4416de08..5ab70d20 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala @@ -2,6 +2,9 @@ package docspell.store.queries object QueryWildcard { + def lower(s: String): String = + apply(s.toLowerCase) + def apply(value: String): String = { def prefix(n: String) = if (n.startsWith("*")) s"%${n.substring(1)}" diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index 26372748..e6cd356c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -6,7 +6,6 @@ import fs2.Stream import docspell.common._ import docspell.store.qb.DSL._ -import docspell.store.qb.TableDef import docspell.store.qb._ import bitpeace.FileMeta @@ -115,16 +114,6 @@ object RAttachment { def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = { import bitpeace.sql._ -// val cols = RFileMeta.Columns.all.map(_.prefix("m")) -// val aId = id.prefix("a") -// val aFileMeta = fileId.prefix("a") -// val mId = RFileMeta.Columns.id.prefix("m") -// -// val from = -// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId) -// val cond = aId.is(attachId) -// -// selectSimple(cols, from, cond).query[FileMeta].option val m = RFileMeta.as("m") val a = RAttachment.as("a") Select( @@ -167,14 +156,6 @@ object RAttachment { attachId: Ident, collective: Ident ): ConnectionIO[Boolean] = { -// val aId = id.prefix("a") -// val aItem = itemId.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val from = -// table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) -// val cond = and(iColl.is(collective), aId.is(attachId)) -// selectCount(id, from, cond).query[Int].unique.map(_ > 0) val a = RAttachment.as("a") val i = RItem.as("i") Select( @@ -189,12 +170,6 @@ object RAttachment { id: Ident, coll: Ident ): ConnectionIO[Vector[RAttachment]] = { -// val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id -// .prefix("i") -// .is(itemId.prefix("a")) ++ -// fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) -// q.query[RAttachment].to[Vector] val a = RAttachment.as("a") val i = RItem.as("i") Select( @@ -210,29 +185,6 @@ object RAttachment { coll: Ident, fileIds: NonEmptyList[Ident] ): ConnectionIO[Vector[RAttachment]] = { - -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val aItem = Columns.itemId.prefix("a") -// val aId = Columns.id.prefix("a") -// val aFile = Columns.fileId.prefix("a") -// val sId = RAttachmentSource.Columns.id.prefix("s") -// val sFile = RAttachmentSource.Columns.fileId.prefix("s") -// val rId = RAttachmentArchive.Columns.id.prefix("r") -// val rFile = RAttachmentArchive.Columns.fileId.prefix("r") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"LEFT JOIN" ++ -// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"LEFT JOIN" ++ -// RAttachmentArchive.table ++ fr"r ON" ++ rId.is(aId) -// -// val cond = and( -// iId.is(id), -// iColl.is(coll), -// or(aFile.isIn(fileIds), sFile.isIn(fileIds), rFile.isIn(fileIds)) -// ) -// -// selectSimple(all.map(_.prefix("a")), from, cond).query[RAttachment].to[Vector] val i = RItem.as("i") val a = RAttachment.as("a") val s = RAttachmentSource.as("s") @@ -255,19 +207,6 @@ object RAttachment { ): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ -// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) -// val afileMeta = fileId.prefix("a") -// val aItem = itemId.prefix("a") -// val mId = RFileMeta.Columns.id.prefix("m") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = -// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) -// val cond = Seq(aItem.is(id), iColl.is(coll)) -// -// selectSimple(cols, from, and(cond)).query[(RAttachment, FileMeta)].to[Vector] val a = RAttachment.as("a") val m = RFileMeta.as("m") val i = RItem.as("i") @@ -283,9 +222,6 @@ object RAttachment { def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ -// val q = -// fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filzemeta m -// WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" val a = RAttachment.as("a") val m = RFileMeta.as("m") Select( @@ -313,24 +249,6 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aItem = Columns.itemId.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val cols = all.map(_.prefix("a")) -// -// coll match { -// case Some(cid) => -// val join = table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) -// val cond = iColl.is(cid) -// selectSimple(cols, join, cond) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// case None => -// selectSimple(cols, table, Fragment.empty) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// } val a = RAttachment.as("a") val i = RItem.as("i") @@ -350,19 +268,6 @@ object RAttachment { } def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aCreated = Columns.created.prefix("a") -// val mId = RAttachmentMeta.Columns.id.prefix("m") -// val mPages = RAttachmentMeta.Columns.pages.prefix("m") -// -// val cols = all.map(_.prefix("a")) -// val join = table ++ fr"a LEFT OUTER JOIN" ++ -// RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) -// val cond = mPages.isNull -// -// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) val a = RAttachment.as("a") val m = RAttachmentMeta.as("m") Select( @@ -377,33 +282,6 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aItem = Columns.itemId.prefix("a") -// val aCreated = Columns.created.prefix("a") -// val pId = RAttachmentPreview.Columns.id.prefix("p") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val cols = all.map(_.prefix("a")) -// val baseJoin = -// table ++ fr"a LEFT OUTER JOIN" ++ -// RAttachmentPreview.table ++ fr"p ON" ++ pId.is(aId) -// -// val baseCond = -// Seq(pId.isNull) -// -// coll match { -// case Some(cid) => -// val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) -// val cond = and(baseCond ++ Seq(iColl.is(cid))) -// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// case None => -// (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// } val a = RAttachment.as("a") val p = RAttachmentPreview.as("p") val i = RItem.as("i") @@ -420,32 +298,11 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aItem = Columns.itemId.prefix("a") -// val aFile = Columns.fileId.prefix("a") -// val sId = RAttachmentSource.Columns.id.prefix("s") -// val sFile = RAttachmentSource.Columns.fileId.prefix("s") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val mId = RFileMeta.Columns.id.prefix("m") -// val mType = RFileMeta.Columns.mimetype.prefix("m") val pdfType = "application/pdf%" -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"INNER JOIN" ++ -// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"INNER JOIN" ++ -// RFileMeta.table ++ fr"m ON" ++ aFile.is(mId) -// val where = coll match { -// case Some(cid) => and(iColl.is(cid), aFile.is(sFile), mType.lowerLike(pdfType)) -// case None => and(aFile.is(sFile), mType.lowerLike(pdfType)) -// } -// selectSimple(all.map(_.prefix("a")), from, where) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) - val a = RAttachment.as("a") - val s = RAttachmentSource.as("s") - val i = RItem.as("i") - val m = RFileMeta.as("m") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val i = RItem.as("i") + val m = RFileMeta.as("m") Select( select(a.all), diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index 5e9f4eb2..cca55b8f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -21,29 +21,29 @@ object RTagItem { val tagId = Column[Ident]("tid", this) val all = NonEmptyList.of[Column[_]](tagItemId, itemId, tagId) } - val t = Table(None) + val T = Table(None) def as(alias: String): Table = Table(Some(alias)) def insert(v: RTagItem): ConnectionIO[Int] = - DML.insert(t, t.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") + DML.insert(T, T.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") def deleteItemTags(item: Ident): ConnectionIO[Int] = - DML.delete(t, t.itemId === item) + DML.delete(T, T.itemId === item) def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] = - DML.delete(t, t.itemId.in(RItem.filterItemsFragment(items, cid))) + DML.delete(T, T.itemId.in(RItem.filterItemsFragment(items, cid))) def deleteTag(tid: Ident): ConnectionIO[Int] = - DML.delete(t, t.tagId === tid) + DML.delete(T, T.tagId === tid) def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] = - run(select(t.all), from(t), t.itemId === item).query[RTagItem].to[Vector] + run(select(T.all), from(T), T.itemId === item).query[RTagItem].to[Vector] def findAllIn(item: Ident, tags: Seq[Ident]): ConnectionIO[Vector[RTagItem]] = NonEmptyList.fromList(tags.toList) match { case Some(nel) => - run(select(t.all), from(t), t.itemId === item && t.tagId.in(nel)) + run(select(T.all), from(T), T.itemId === item && T.tagId.in(nel)) .query[RTagItem] .to[Vector] case None => @@ -55,7 +55,7 @@ object RTagItem { case None => 0.pure[ConnectionIO] case Some(nel) => - DML.delete(t, t.itemId === item && t.tagId.in(nel)) + DML.delete(T, T.itemId === item && T.tagId.in(nel)) } def setAllTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = @@ -67,8 +67,8 @@ object RTagItem { ) n <- DML .insertMany( - t, - t.all, + T, + T.all, entities.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") ) } yield n diff --git a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala index 45ef618a..9b7d8b77 100644 --- a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala +++ b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala @@ -4,8 +4,7 @@ import cats.data.NonEmptyList import docspell.common._ import docspell.store.qb.DSL._ - -import doobie._ +import docspell.store.qb.{Condition, Select} /** A helper class combining information from `RTag` and `RTagItem`. * This is not a "record", there is no corresponding table. @@ -20,32 +19,80 @@ case class TagItemName( ) object TagItemName { + private val ti = RTagItem.as("ti") + private val t = RTag.as("t") - def itemsInCategory(cats: NonEmptyList[String]): Fragment = { - val catsLower = cats.map(_.toLowerCase) - val ti = RTagItem.as("ti") - val t = RTag.as("t") - val join = from(t).innerJoin(ti, t.tid === ti.tagId) - if (cats.tail.isEmpty) - run(select(ti.itemId), join, t.category.likes(catsLower.head)) - else - run(select(ti.itemId), join, t.category.inLower(cats)) - } + private val taggedItems = + from(t).innerJoin(ti, t.tid === ti.tagId) - def itemsWithTagOrCategory(tags: List[Ident], cats: List[String]): Fragment = { + private def orTags(tags: NonEmptyList[Ident]): Condition = + ti.tagId.in(tags) + + private def orCategory(cats: NonEmptyList[String]): Condition = + t.category.inLower(cats) + + def itemsInEitherCategory(cats: NonEmptyList[String]): Select = + Select(ti.itemId.s, taggedItems, orCategory(cats)).distinct + + def itemsInAllCategories(cats: NonEmptyList[String]): Select = + intersect( + cats.map(cat => Select(ti.itemId.s, taggedItems, t.category === cat).distinct) + ) + + def itemsWithEitherTag(tags: NonEmptyList[Ident]): Select = + Select(ti.itemId.s, from(ti), orTags(tags)).distinct + + def itemsWithAllTags(tags: NonEmptyList[Ident]): Select = + intersect(tags.map(tid => Select(ti.itemId.s, from(ti), ti.tagId === tid).distinct)) + + def itemsWithEitherTagOrCategory( + tags: NonEmptyList[Ident], + cats: NonEmptyList[String] + ): Select = + Select(ti.itemId.s, taggedItems, orTags(tags) || orCategory(cats)) + + def itemsWithAllTagAndCategory( + tags: NonEmptyList[Ident], + cats: NonEmptyList[String] + ): Select = + Select( + ti.itemId.s, + from(ti), + ti.itemId.in(itemsWithAllTags(tags)) && + ti.itemId.in(itemsInAllCategories(cats)) + ) + + def itemsWithEitherTagOrCategory( + tags: List[Ident], + cats: List[String] + ): Option[Select] = { val catsLower = cats.map(_.toLowerCase) - val ti = RTagItem.as("ti") - val t = RTag.as("t") - val join = from(t).innerJoin(ti, t.tid === ti.tagId) (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match { case (Some(tagNel), Some(catNel)) => - run(select(ti.itemId), join, t.tid.in(tagNel) || t.category.inLower(catNel)) + Some(itemsWithEitherTagOrCategory(tagNel, catNel)) case (Some(tagNel), None) => - run(select(ti.itemId), join, t.tid.in(tagNel)) + Some(itemsWithEitherTag(tagNel)) case (None, Some(catNel)) => - run(select(ti.itemId), join, t.category.inLower(catNel)) + Some(itemsInEitherCategory(catNel)) case (None, None) => - Fragment.empty + None + } + } + + def itemsWithAllTagAndCategory( + tags: List[Ident], + cats: List[String] + ): Option[Select] = { + val catsLower = cats.map(_.toLowerCase) + (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match { + case (Some(tagNel), Some(catNel)) => + Some(itemsWithAllTagAndCategory(tagNel, catNel)) + case (Some(tagNel), None) => + Some(itemsWithAllTags(tagNel)) + case (None, Some(catNel)) => + Some(itemsInAllCategories(catNel)) + case (None, None) => + None } } } diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala index e35c73d9..7d5dc64d 100644 --- a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -48,14 +48,14 @@ object QueryBuilderTest extends SimpleTestSuite { assertEquals(f, FromExpr.From(c)) assertEquals(2, joins.size) joins.head match { - case Join.InnerJoin(tbl, cond) => + case FromExpr.Join.InnerJoin(FromExpr.Relation.Table(tbl), cond) => assertEquals(tbl, owner) assertEquals(cond, c.ownerId === owner.id) case _ => fail("Unexpected join result") } joins.tail.head match { - case Join.LeftJoin(tbl, cond) => + case FromExpr.Join.LeftJoin(FromExpr.Relation.Table(tbl), cond) => assertEquals(tbl, lecturer) assertEquals(cond, c.lecturerId === lecturer.id) case _ => diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala new file mode 100644 index 00000000..4a3f062e --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala @@ -0,0 +1,19 @@ +package docspell.store.qb.impl + +import minitest._ +import docspell.store.qb._ +import docspell.store.qb.DSL._ +import docspell.store.qb.model.{CourseRecord, PersonRecord} + +object DSLTest extends SimpleTestSuite { + + val course = CourseRecord.as("c") + val person = PersonRecord.as("p") + + test("and") { + val c = course.lessons > 4 && person.id === 3 && person.name.like("%a%") + val expect = + Condition.And(course.lessons > 4, person.id === 3, person.name.like("%a%")) + assertEquals(c, expect) + } +}