From b338f18e9859ca0d1996504f1664d20acf4271cc Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 6 Dec 2020 22:53:18 +0100 Subject: [PATCH 01/38] Remove redundant fileCount from search result --- modules/restapi/src/main/resources/docspell-openapi.yml | 4 ---- .../main/scala/docspell/restserver/conv/Conversions.scala | 1 - modules/webapp/src/main/elm/Comp/ItemCard.elm | 6 +++--- modules/webapp/src/main/elm/Data/ItemTemplate.elm | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 11d78d42..cc70f5f0 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -5079,7 +5079,6 @@ components: - state - date - source - - fileCount - tags properties: id: @@ -5113,9 +5112,6 @@ components: $ref: "#/components/schemas/IdName" folder: $ref: "#/components/schemas/IdName" - fileCount: - type: integer - format: int32 attachments: type: array items: diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index bdba7bf6..ece9ae45 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -213,7 +213,6 @@ trait Conversions { i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.folder.map(mkIdName), - i.fileCount, Nil, //attachments Nil, //tags Nil, //customfields diff --git a/modules/webapp/src/main/elm/Comp/ItemCard.elm b/modules/webapp/src/main/elm/Comp/ItemCard.elm index 11213bf9..0de41d9e 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCard.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCard.elm @@ -473,12 +473,12 @@ previewMenu settings model item mainAttach = [ i [ class "eye icon" ] [] ] in - if pageCount == 0 || (item.fileCount == 1 && pageCount == 1) then + if pageCount == 0 || (List.length item.attachments == 1 && pageCount == 1) then div [ class "card-attachment-nav" ] [ gotoFileBtn ] - else if item.fileCount == 1 then + else if List.length item.attachments == 1 then div [ class "card-attachment-nav" ] [ div [ class "ui small top attached basic icon buttons" ] [ gotoFileBtn @@ -510,7 +510,7 @@ previewMenu settings model item mainAttach = |> String.fromInt |> text , text "/" - , text (String.fromInt item.fileCount) + , text (List.length item.attachments |> String.fromInt) , text ", " , text (String.fromInt pageCount) , text "p." diff --git a/modules/webapp/src/main/elm/Data/ItemTemplate.elm b/modules/webapp/src/main/elm/Data/ItemTemplate.elm index 957e9949..30fbf43d 100644 --- a/modules/webapp/src/main/elm/Data/ItemTemplate.elm +++ b/modules/webapp/src/main/elm/Data/ItemTemplate.elm @@ -224,7 +224,7 @@ concerning = fileCount : ItemTemplate fileCount = - ItemTemplate (.fileCount >> String.fromInt) + ItemTemplate (.attachments >> List.length >> String.fromInt) From 2dbb1db2fd14b4c2c5c42ba54d6196c27cbde761 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 7 Dec 2020 19:30:10 +0100 Subject: [PATCH 02/38] Initial outline for a simple query builder --- .../main/scala/docspell/store/qb/Column.scala | 5 ++ .../scala/docspell/store/qb/Condition.scala | 20 +++++ .../scala/docspell/store/qb/DBFunction.scala | 26 ++++++ .../main/scala/docspell/store/qb/DML.scala | 62 +++++++++++++++ .../main/scala/docspell/store/qb/DSL.scala | 79 +++++++++++++++++++ .../scala/docspell/store/qb/FromExpr.scala | 28 +++++++ .../scala/docspell/store/qb/GroupBy.scala | 3 + .../main/scala/docspell/store/qb/Join.scala | 10 +++ .../scala/docspell/store/qb/Operator.scala | 14 ++++ .../scala/docspell/store/qb/OrderBy.scala | 14 ++++ .../main/scala/docspell/store/qb/Select.scala | 49 ++++++++++++ .../scala/docspell/store/qb/SelectExpr.scala | 11 +++ .../main/scala/docspell/store/qb/Setter.scala | 17 ++++ .../scala/docspell/store/qb/TableDef.scala | 7 ++ .../store/qb/impl/ConditionBuilder.scala | 74 +++++++++++++++++ .../docspell/store/qb/impl/DoobieQuery.scala | 72 +++++++++++++++++ .../store/qb/impl/FromExprBuilder.scala | 36 +++++++++ .../store/qb/impl/SelectExprBuilder.scala | 30 +++++++ .../docspell/store/qb/QueryBuilderTest.scala | 37 +++++++++ .../store/qb/impl/DoobieQueryTest.scala | 50 ++++++++++++ .../store/qb/model/CourseRecord.scala | 34 ++++++++ .../store/qb/model/PersonRecord.scala | 38 +++++++++ 22 files changed, 716 insertions(+) create mode 100644 modules/store/src/main/scala/docspell/store/qb/Column.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/Condition.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/DBFunction.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/DML.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/DSL.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/FromExpr.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/GroupBy.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/Join.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/Operator.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/OrderBy.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/Select.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/Setter.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/TableDef.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala create mode 100644 modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala create mode 100644 modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala create mode 100644 modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala create mode 100644 modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala new file mode 100644 index 00000000..3f3fa1ab --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -0,0 +1,5 @@ +package docspell.store.qb + +case class Column[A](name: String, table: TableDef, alias: Option[String] = None) + +object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala new file mode 100644 index 00000000..45f9a4c7 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -0,0 +1,20 @@ +package docspell.store.qb + +import doobie._ + +sealed trait Condition {} + +object Condition { + + case class CompareVal[A](column: Column[A], op: Operator, value: A)(implicit + val P: Put[A] + ) extends Condition + + case class CompareCol[A](col1: Column[A], op: Operator, col2: Column[A]) + extends Condition + + case class And(c: Condition, cs: Vector[Condition]) extends Condition + case class Or(c: Condition, cs: Vector[Condition]) extends Condition + case class Not(c: Condition) extends Condition + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala new file mode 100644 index 00000000..1e597efc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -0,0 +1,26 @@ +package docspell.store.qb + +sealed trait DBFunction { + def alias: String + + def as(alias: String): DBFunction +} + +object DBFunction { + + def countAllAs(alias: String) = + CountAll(alias) + + def countAs[A](column: Column[A], alias: String): DBFunction = + Count(column, alias) + + case class CountAll(alias: String) extends DBFunction { + def as(a: String) = + copy(alias = a) + } + + case class Count(column: Column[_], alias: String) extends DBFunction { + def as(a: String) = + copy(alias = a) + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/DML.scala b/modules/store/src/main/scala/docspell/store/qb/DML.scala new file mode 100644 index 00000000..dbbe1c79 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/DML.scala @@ -0,0 +1,62 @@ +package docspell.store.qb + +import docspell.store.qb.impl._ + +import doobie._ +import doobie.implicits._ + +object DML { + private val comma = fr"," + + def delete(table: TableDef, cond: Condition): Fragment = + fr"DELETE FROM" ++ FromExprBuilder.buildTable(table) ++ fr"WHERE" ++ ConditionBuilder + .build(cond) + + def insert(table: TableDef, cols: Seq[Column[_]], values: Fragment): Fragment = + fr"INSERT INTO" ++ FromExprBuilder.buildTable(table) ++ sql"(" ++ + cols + .map(SelectExprBuilder.columnNoPrefix) + .reduce(_ ++ comma ++ _) ++ fr") VALUES (" ++ + values ++ fr")" + + def update( + table: TableDef, + cond: Condition, + setter: Seq[Setter[_]] + ): ConnectionIO[Int] = + update(table, Some(cond), setter).update.run + + def update( + table: TableDef, + cond: Option[Condition], + setter: Seq[Setter[_]] + ): Fragment = { + val condFrag = cond.map(DoobieQuery.cond).getOrElse(Fragment.empty) + fr"UPDATE" ++ FromExprBuilder.buildTable(table) ++ fr"SET" ++ + setter + .map(s => buildSetter(s)) + .reduce(_ ++ comma ++ _) ++ + condFrag + } + + private def buildSetter[A](setter: Setter[A]): Fragment = + setter match { + case s @ Setter.SetValue(column, value) => + SelectExprBuilder.columnNoPrefix(column) ++ fr" =" ++ ConditionBuilder.buildValue( + value + )(s.P) + + case s @ Setter.SetOptValue(column, optValue) => + SelectExprBuilder.columnNoPrefix(column) ++ fr" =" ++ ConditionBuilder + .buildOptValue( + optValue + )(s.P) + + case Setter.Increment(column, amount) => + val colFrag = SelectExprBuilder.columnNoPrefix(column) + colFrag ++ fr" =" ++ colFrag ++ fr" + $amount" + } + + def set(s: Setter[_], more: Setter[_]*): Seq[Setter[_]] = + more :+ s +} diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala new file mode 100644 index 00000000..63eb7083 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -0,0 +1,79 @@ +package docspell.store.qb + +import docspell.store.impl.DoobieMeta +import docspell.store.qb.impl.DoobieQuery + +import doobie.{Fragment, Put} + +trait DSL extends DoobieMeta { + + def run(projection: Seq[SelectExpr], from: FromExpr, where: Condition): Fragment = + DoobieQuery(Select(projection, from, where)) + + def select(dbf: DBFunction): Seq[SelectExpr] = + Seq(SelectExpr.SelectFun(dbf)) + + def select(c: Column[_], cs: Column[_]*): Seq[SelectExpr] = + select(c :: cs.toList) + + def select(seq: Seq[Column[_]], seqs: Seq[Column[_]]*): Seq[SelectExpr] = + (seq ++ seqs.flatten).map(SelectExpr.SelectColumn.apply) + + def from(table: TableDef): FromExpr = + FromExpr.From(table) + + def count(c: Column[_]): DBFunction = + DBFunction.Count(c, "cn") + + def and(c: Condition, cs: Condition*): Condition = + Condition.And(c, cs.toVector) + + def or(c: Condition, cs: Condition*): Condition = + Condition.Or(c, cs.toVector) + + def where(c: Condition, cs: Condition*): Condition = + and(c, cs: _*) + + implicit final class ColumnOps[A](col: Column[A]) { + + def setTo(value: A)(implicit P: Put[A]): Setter[A] = + Setter.SetValue(col, value) + + def setTo(value: Option[A])(implicit P: Put[A]): Setter[Option[A]] = + Setter.SetOptValue(col, value) + + def increment(amount: Int): Setter[A] = + Setter.Increment(col, amount) + + def asc: OrderBy = + OrderBy(SelectExpr.SelectColumn(col), OrderBy.OrderType.Asc) + + def desc: OrderBy = + OrderBy(SelectExpr.SelectColumn(col), OrderBy.OrderType.Desc) + + def ===(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Eq, value) + + def like(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.LowerLike, value) + + def <=(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Lte, value) + + def >=(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Gte, value) + + def >(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Gt, value) + + def <(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Lt, value) + + def ===(other: Column[A]): Condition = + Condition.CompareCol(col, Operator.Eq, other) + + } + +} + +object DSL extends DSL diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala new file mode 100644 index 00000000..f066cccc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -0,0 +1,28 @@ +package docspell.store.qb + +sealed trait FromExpr { + + def innerJoin(other: TableDef, on: Condition): FromExpr + + def leftJoin(other: TableDef, on: Condition): FromExpr +} + +object FromExpr { + + case class From(table: TableDef) extends FromExpr { + def innerJoin(other: TableDef, on: Condition): Joined = + Joined(this, Vector(Join.InnerJoin(other, on))) + + def leftJoin(other: TableDef, on: Condition): Joined = + Joined(this, Vector(Join.LeftJoin(other, on))) + } + + case class Joined(from: From, joins: Vector[Join]) extends FromExpr { + def innerJoin(other: TableDef, on: Condition): Joined = + Joined(from, joins :+ Join.InnerJoin(other, on)) + + def leftJoin(other: TableDef, on: Condition): Joined = + Joined(from, joins :+ Join.LeftJoin(other, on)) + + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala new file mode 100644 index 00000000..a12e6448 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala @@ -0,0 +1,3 @@ +package docspell.store.qb + +case class GroupBy(name: SelectExpr, names: Vector[SelectExpr], having: Option[Condition]) diff --git a/modules/store/src/main/scala/docspell/store/qb/Join.scala b/modules/store/src/main/scala/docspell/store/qb/Join.scala new file mode 100644 index 00000000..a51a3b70 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Join.scala @@ -0,0 +1,10 @@ +package docspell.store.qb + +sealed trait Join + +object Join { + + case class InnerJoin(table: TableDef, cond: Condition) extends Join + + case class LeftJoin(table: TableDef, cond: Condition) extends Join +} diff --git a/modules/store/src/main/scala/docspell/store/qb/Operator.scala b/modules/store/src/main/scala/docspell/store/qb/Operator.scala new file mode 100644 index 00000000..c05559ca --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Operator.scala @@ -0,0 +1,14 @@ +package docspell.store.qb + +sealed trait Operator + +object Operator { + + case object Eq extends Operator + case object Gt extends Operator + case object Lt extends Operator + case object Gte extends Operator + case object Lte extends Operator + case object LowerLike extends Operator + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala new file mode 100644 index 00000000..24d42cd4 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala @@ -0,0 +1,14 @@ +package docspell.store.qb + +import docspell.store.qb.OrderBy.OrderType + +final case class OrderBy(expr: SelectExpr, orderType: OrderType) + +object OrderBy { + + sealed trait OrderType + object OrderType { + case object Asc extends OrderType + case object Desc extends OrderType + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala new file mode 100644 index 00000000..accb90a0 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -0,0 +1,49 @@ +package docspell.store.qb + +import docspell.store.qb.impl.DoobieQuery + +import doobie._ + +sealed trait Select { + def distinct: Fragment = + DoobieQuery.distinct(this) + + def run: Fragment = + DoobieQuery(this) + + def orderBy(ob: OrderBy, obs: OrderBy*): Select = + Select.Ordered(this, ob, obs.toVector) + + def orderBy(c: Column[_]): Select = + orderBy(OrderBy(SelectExpr.SelectColumn(c), OrderBy.OrderType.Asc)) +} + +object Select { + + def apply( + projection: Seq[SelectExpr], + from: FromExpr, + where: Condition + ) = SimpleSelect(projection, from, Some(where), None) + + def apply( + projection: Seq[SelectExpr], + from: FromExpr, + where: Option[Condition] = None, + groupBy: Option[GroupBy] = None + ) = SimpleSelect(projection, from, where, groupBy) + + case class SimpleSelect( + projection: Seq[SelectExpr], + from: FromExpr, + where: Option[Condition], + groupBy: Option[GroupBy] + ) extends Select + + case class Union(q: Select, qs: Vector[Select]) extends Select + + case class Intersect(q: Select, qs: Vector[Select]) extends Select + + case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) + extends Select +} diff --git a/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala new file mode 100644 index 00000000..1ccb3b90 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala @@ -0,0 +1,11 @@ +package docspell.store.qb + +sealed trait SelectExpr + +object SelectExpr { + + case class SelectColumn(column: Column[_]) extends SelectExpr + + case class SelectFun(fun: DBFunction) extends SelectExpr + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/Setter.scala b/modules/store/src/main/scala/docspell/store/qb/Setter.scala new file mode 100644 index 00000000..d86af800 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/Setter.scala @@ -0,0 +1,17 @@ +package docspell.store.qb + +import doobie._ + +sealed trait Setter[A] + +object Setter { + + case class SetOptValue[A](column: Column[A], value: Option[A])(implicit val P: Put[A]) + extends Setter[Option[A]] + + case class SetValue[A](column: Column[A], value: A)(implicit val P: Put[A]) + extends Setter[A] + + case class Increment[A](column: Column[A], amount: Int) extends Setter[A] + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala new file mode 100644 index 00000000..13da97cf --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala @@ -0,0 +1,7 @@ +package docspell.store.qb + +trait TableDef { + def tableName: String + + def alias: Option[String] +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala new file mode 100644 index 00000000..5a37733f --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -0,0 +1,74 @@ +package docspell.store.qb.impl + +import docspell.store.qb._ + +import _root_.doobie.implicits._ +import _root_.doobie.{Query => _, _} + +object ConditionBuilder { + val or = fr"OR" + val and = fr"AND" + val parenOpen = Fragment.const0("(") + val parenClose = Fragment.const0(")") + + def build(expr: Condition): Fragment = + expr match { + case c @ Condition.CompareVal(col, op, value) => + val opFrag = operator(op) + val valFrag = buildValue(value)(c.P) + val colFrag = op match { + case Operator.LowerLike => + lower(col) + case _ => + SelectExprBuilder.column(col) + } + colFrag ++ opFrag ++ valFrag + + case Condition.CompareCol(c1, op, c2) => + val (c1Frag, c2Frag) = op match { + case Operator.LowerLike => + (lower(c1), lower(c2)) + case _ => + (SelectExprBuilder.column(c1), SelectExprBuilder.column(c2)) + } + c1Frag ++ operator(op) ++ c2Frag + + case Condition.And(c, cs) => + val inner = cs.prepended(c).map(build).reduce(_ ++ and ++ _) + if (cs.isEmpty) inner + else parenOpen ++ inner ++ parenClose + + case Condition.Or(c, cs) => + val inner = cs.prepended(c).map(build).reduce(_ ++ or ++ _) + if (cs.isEmpty) inner + else parenOpen ++ inner ++ parenClose + + case Condition.Not(c) => + fr"NOT" ++ build(c) + } + + def operator(op: Operator): Fragment = + op match { + case Operator.Eq => + fr" =" + case Operator.Gt => + fr" >" + case Operator.Lt => + fr" <" + case Operator.Gte => + fr" >=" + case Operator.Lte => + fr" <=" + case Operator.LowerLike => + fr" LIKE" + } + + def buildValue[A: Put](v: A): Fragment = + fr"$v" + + def buildOptValue[A: Put](v: Option[A]): Fragment = + fr"$v" + + def lower(col: Column[_]): Fragment = + Fragment.const0("LOWER(") ++ SelectExprBuilder.column(col) ++ sql")" +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala new file mode 100644 index 00000000..e20d9e72 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala @@ -0,0 +1,72 @@ +package docspell.store.qb.impl + +import docspell.store.qb._ + +import _root_.doobie.implicits._ +import _root_.doobie.{Query => _, _} + +object DoobieQuery { + val comma = fr"," + val asc = fr" ASC" + val desc = fr" DESC" + val intersect = fr"INTERSECT" + val union = fr"UNION ALL" + + def apply(q: Select): Fragment = + build(false)(q) + + def distinct(q: Select): Fragment = + build(true)(q) + + def build(distinct: Boolean)(q: Select): Fragment = + q match { + case sq: Select.SimpleSelect => + val sel = if (distinct) fr"SELECT DISTINCT" else fr"SELECT" + sel ++ buildSimple(sq) + + case Select.Union(q, qs) => + qs.prepended(q).map(build(false)).reduce(_ ++ union ++ _) + + case Select.Intersect(q, qs) => + qs.prepended(q).map(build(false)).reduce(_ ++ intersect ++ _) + + case Select.Ordered(q, ob, obs) => + val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _) + build(distinct)(q) ++ fr"ORDER BY" ++ order + + } + + def buildSimple(sq: Select.SimpleSelect): Fragment = { + val f0 = sq.projection.map(selectExpr).reduce(_ ++ comma ++ _) + val f1 = fromExpr(sq.from) + val f2 = sq.where.map(cond).getOrElse(Fragment.empty) + val f3 = sq.groupBy.map(groupBy).getOrElse(Fragment.empty) + f0 ++ f1 ++ f2 ++ f3 + } + + def orderBy(ob: OrderBy): Fragment = { + val f1 = selectExpr(ob.expr) + val f2 = ob.orderType match { + case OrderBy.OrderType.Asc => + asc + case OrderBy.OrderType.Desc => + desc + } + f1 ++ f2 + } + + def selectExpr(se: SelectExpr): Fragment = + SelectExprBuilder.build(se) + + def fromExpr(fr: FromExpr): Fragment = + FromExprBuilder.build(fr) + + def cond(c: Condition): Fragment = + fr" WHERE" ++ ConditionBuilder.build(c) + + def groupBy(gb: GroupBy): Fragment = { + val f0 = gb.names.prepended(gb.name).map(selectExpr).reduce(_ ++ comma ++ _) + val f1 = gb.having.map(cond).getOrElse(Fragment.empty) + fr"GROUP BY" ++ f0 ++ f1 + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala new file mode 100644 index 00000000..39e10864 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala @@ -0,0 +1,36 @@ +package docspell.store.qb.impl + +import docspell.store.qb._ + +import _root_.doobie.implicits._ +import _root_.doobie.{Query => _, _} + +object FromExprBuilder { + + def build(expr: FromExpr): Fragment = + expr match { + case FromExpr.From(table) => + fr" FROM" ++ buildTable(table) + + case FromExpr.Joined(from, joins) => + build(from) ++ + joins.map(buildJoin).foldLeft(Fragment.empty)(_ ++ _) + } + + def buildTable(table: TableDef): Fragment = + Fragment.const(table.tableName) ++ table.alias + .map(a => Fragment.const0(a)) + .getOrElse(Fragment.empty) + + def buildJoin(join: Join): Fragment = + join match { + case Join.InnerJoin(table, cond) => + val c = fr" ON" ++ ConditionBuilder.build(cond) + fr" INNER JOIN" ++ buildTable(table) ++ c + + case Join.LeftJoin(table, cond) => + val c = fr" ON" ++ ConditionBuilder.build(cond) + fr" LEFT JOIN" ++ buildTable(table) ++ c + } + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala new file mode 100644 index 00000000..e60308bd --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala @@ -0,0 +1,30 @@ +package docspell.store.qb.impl + +import docspell.store.qb._ + +import _root_.doobie.implicits._ +import _root_.doobie.{Query => _, _} + +object SelectExprBuilder { + + def build(expr: SelectExpr): Fragment = + expr match { + case SelectExpr.SelectColumn(col) => + column(col) + + case SelectExpr.SelectFun(DBFunction.CountAll(alias)) => + fr"COUNT(*) AS" ++ Fragment.const(alias) + + case SelectExpr.SelectFun(DBFunction.Count(col, alias)) => + fr"COUNT(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) + } + + def column(col: Column[_]): Fragment = { + val prefix = + Fragment.const0(col.table.alias.getOrElse(col.table.tableName)) + prefix ++ Fragment.const0(".") ++ Fragment.const0(col.name) + } + + def columnNoPrefix(col: Column[_]): Fragment = + Fragment.const0(col.name) +} diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala new file mode 100644 index 00000000..f7912f4a --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -0,0 +1,37 @@ +package docspell.store.qb + +import minitest._ +import docspell.store.qb.model._ +import docspell.store.qb.DSL._ + +object QueryBuilderTest extends SimpleTestSuite { + + test("simple") { + val c = CourseRecord.as("c") + val owner = PersonRecord.as("p1") + val lecturer = PersonRecord.as("p2") + + val proj = select(c.all, owner.all, lecturer.all) + + val tables = + from(c) + .innerJoin(owner, c.ownerId === owner.id) + .leftJoin(lecturer, c.lecturerId === lecturer.id) + + val cond = + where( + c.name.like("%scala%"), + c.lessons <= 15, + or( + owner.name.like("%"), + lecturer.id >= 1 + ) + ) + + // val order = + // orderBy(c.name.asc) + + val q = Select(proj, tables, cond) + println(q) + } +} diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala new file mode 100644 index 00000000..9b38d94f --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala @@ -0,0 +1,50 @@ +package docspell.store.qb.impl + +import minitest._ +import docspell.store.qb._ +import docspell.store.qb.model._ +import docspell.store.qb.DSL._ +import docspell.common._ + +object DoobieQueryTest extends SimpleTestSuite { + + test("basic fragment") { + val c = CourseRecord.as("c") + val owner = PersonRecord.as("o") + val lecturer = PersonRecord.as("l") + + val proj = select(c.all) + val table = from(c) + .innerJoin(owner, c.ownerId === owner.id) + .leftJoin(lecturer, c.lecturerId === lecturer.id) + val cond = where( + c.name.like("%test%"), + owner.name === "Harald" + ) + + val q = Select(proj, table, cond) + val frag = DoobieQuery.select(q) + assertEquals( + frag.toString, + """Fragment("SELECT c.id, c.name, c.owner_id, c.lecturer_id, c.lessons FROM course c INNER JOIN person o ON c.owner_id = o.id LEFT JOIN person l ON c.lecturer_id = l.id WHERE (LOWER(c.name) LIKE ? AND o.name = ? )")""" + ) + } + + test("basic update") { + val p = PersonRecord.table + + val update = PersonRecord.update(p.name.set("john"), p.id.set(15L)).where(p.id >= 2) + + println(DoobieQuery.update(update)) + + } + + test("basic insert") { + val p = PersonRecord(1, "John", Timestamp.Epoch) + + val insert = PersonRecord.insertAll(p) + + println(insert) + + } +} diff --git a/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala new file mode 100644 index 00000000..867a41de --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala @@ -0,0 +1,34 @@ +package docspell.store.qb.model + +import docspell.store.qb._ + +case class CourseRecord( + id: Long, + name: String, + ownerId: Long, + lecturerId: Option[Long], + lessons: Int +) + +object CourseRecord { + + final case class Table(alias: Option[String]) extends TableDef { + + override val tableName = "course" + + val id = Column[Long]("id", this) + val name = Column[String]("name", this) + val ownerId = Column[Long]("owner_id", this) + val lecturerId = Column[Long]("lecturer_id", this) + val lessons = Column[Int]("lessons", this) + + val all = List(id, name, ownerId, lecturerId, lessons) + } + + def as(alias: String): Table = + Table(Some(alias)) + + def update: UpdateTable = + UpdateTable(Table(None), None, Seq.empty) + +} diff --git a/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala new file mode 100644 index 00000000..bfcee143 --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala @@ -0,0 +1,38 @@ +package docspell.store.qb.model + +import docspell.store.qb._ +import docspell.common._ +import doobie.implicits._ +import docspell.store.impl.DoobieMeta._ +import doobie._ + +case class PersonRecord(id: Long, name: String, created: Timestamp) + +object PersonRecord { + + final case class Table(alias: Option[String]) extends TableDef { + + val tableName = "person" + + val id = Column[Long]("id", this) + val name = Column[String]("name", this) + val created = Column[Timestamp]("created", this) + + val all = List(id, name, created) + } + + def as(alias: String): Table = + Table(Some(alias)) + + def table: Table = Table(None) + + def update(set: UpdateTable.Setter[_], sets: UpdateTable.Setter[_]*): UpdateTable = + UpdateTable(table, None, sets :+ set) + + def insertAll(v: PersonRecord): ConnectionIO[Int] = + InsertTable( + table, + table.all, + fr"${v.id},${v.name},${v.created}" + ).toFragment.update.run +} From adee496b77ac2cde5988c809e0b77c9c1940f12a Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 8 Dec 2020 21:04:11 +0100 Subject: [PATCH 03/38] Convert source record --- .../restserver/routes/ItemRoutes.scala | 4 +- .../docspell/store/records/RSource.scala | 104 ++++++++++-------- .../docspell/store/records/SourceData.scala | 6 +- .../docspell/store/qb/QueryBuilderTest.scala | 38 ++++++- .../store/qb/impl/DoobieQueryTest.scala | 20 +--- .../store/qb/model/CourseRecord.scala | 3 - .../store/qb/model/PersonRecord.scala | 14 --- 7 files changed, 98 insertions(+), 91 deletions(-) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index 1a088a39..9f39d180 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -1,17 +1,17 @@ package docspell.restserver.routes +import cats.Monoid import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ -import cats.Monoid import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OFulltext import docspell.backend.ops.OItemSearch.Batch -import docspell.common.syntax.all._ import docspell.common._ +import docspell.common.syntax.all._ import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala index ea7a0c60..3e126df0 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -1,8 +1,8 @@ package docspell.store.records import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -26,20 +26,19 @@ case class RSource( object RSource { - val table = fr"source" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "source" - object Columns { - - val sid = Column("sid") - val cid = Column("cid") - val abbrev = Column("abbrev") - val description = Column("description") - val counter = Column("counter") - val enabled = Column("enabled") - val priority = Column("priority") - val created = Column("created") - val folder = Column("folder_id") - val fileFilter = Column("file_filter") + val sid = Column[Ident]("sid", this) + val cid = Column[Ident]("cid", this) + val abbrev = Column[String]("abbrev", this) + val description = Column[String]("description", this) + val counter = Column[Int]("counter", this) + val enabled = Column[Boolean]("enabled", this) + val priority = Column[Priority]("priority", this) + val created = Column[Timestamp]("created", this) + val folder = Column[Ident]("folder_id", this) + val fileFilter = Column[Glob]("file_filter", this) val all = List( @@ -56,48 +55,54 @@ object RSource { ) } - import Columns._ + def as(alias: String): Table = + Table(Some(alias)) + + val table = Table(None) def insert(v: RSource): ConnectionIO[Int] = { - val sql = insertRow( + val sql = DML.insert( table, - all, + table.all, fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}" ) sql.update.run } - def updateNoCounter(v: RSource): ConnectionIO[Int] = { - val sql = updateRow( + def updateNoCounter(v: RSource): ConnectionIO[Int] = + DML.update( table, - and(sid.is(v.sid), cid.is(v.cid)), - commas( - cid.setTo(v.cid), - abbrev.setTo(v.abbrev), - description.setTo(v.description), - enabled.setTo(v.enabled), - priority.setTo(v.priority), - folder.setTo(v.folderId), - fileFilter.setTo(v.fileFilter) + where(table.sid === v.sid, table.cid === v.cid), + DML.set( + table.cid.setTo(v.cid), + table.abbrev.setTo(v.abbrev), + table.description.setTo(v.description), + table.enabled.setTo(v.enabled), + table.priority.setTo(v.priority), + table.folder.setTo(v.folderId), + table.fileFilter.setTo(v.fileFilter) ) ) - sql.update.run - } def incrementCounter(source: String, coll: Ident): ConnectionIO[Int] = - updateRow( - table, - and(abbrev.is(source), cid.is(coll)), - counter.f ++ fr"=" ++ counter.f ++ fr"+ 1" - ).update.run + DML + .update( + table, + where(table.abbrev === source, table.cid === coll), + DML.set(table.counter.increment(1)) + ) def existsById(id: Ident): ConnectionIO[Boolean] = { - val sql = selectCount(sid, table, sid.is(id)) + val sql = run(select(count(table.sid)), from(table), where(table.sid === id)) sql.query[Int].unique.map(_ > 0) } def existsByAbbrev(coll: Ident, abb: String): ConnectionIO[Boolean] = { - val sql = selectCount(sid, table, and(cid.is(coll), abbrev.is(abb))) + val sql = run( + select(count(table.sid)), + from(table), + where(table.cid === coll, table.abbrev === abb) + ) sql.query[Int].unique.map(_ > 0) } @@ -105,25 +110,34 @@ object RSource { findEnabledSql(id).query[RSource].option private[records] def findEnabledSql(id: Ident): Fragment = - selectSimple(all, table, and(sid.is(id), enabled.is(true))) + run(select(table.all), from(table), where(table.sid === id, table.enabled === true)) def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option + run(select(table.cid), from(table), table.sid === sourceId).query[Ident].option def findAll( coll: Ident, - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[RSource]] = findAllSql(coll, order).query[RSource].to[Vector] - private[records] def findAllSql(coll: Ident, order: Columns.type => Column): Fragment = - selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) + private[records] def findAllSql( + coll: Ident, + order: Table => Column[_] + ): Fragment = { + val t = RSource.as("s") + Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).run + } def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run + DML.delete(table, where(table.sid === sourceId, table.cid === coll)).update.run def removeFolder(folderId: Ident): ConnectionIO[Int] = { val empty: Option[Ident] = None - updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run + DML.update( + table, + where(table.folder === folderId), + DML.set(table.folder.setTo(empty)) + ) } } diff --git a/modules/store/src/main/scala/docspell/store/records/SourceData.scala b/modules/store/src/main/scala/docspell/store/records/SourceData.scala index 8ce65f33..ba8da051 100644 --- a/modules/store/src/main/scala/docspell/store/records/SourceData.scala +++ b/modules/store/src/main/scala/docspell/store/records/SourceData.scala @@ -5,8 +5,8 @@ import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.impl.DoobieMeta._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -22,7 +22,7 @@ object SourceData { def findAll( coll: Ident, - order: RSource.Columns.type => Column + order: RSource.Table => Column[_] ): Stream[ConnectionIO, SourceData] = findAllWithTags(RSource.findAllSql(coll, order).query[RSource].stream) diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala index f7912f4a..f13550a7 100644 --- a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -1,6 +1,7 @@ package docspell.store.qb import minitest._ +import docspell.store.qb._ import docspell.store.qb.model._ import docspell.store.qb.DSL._ @@ -28,10 +29,37 @@ object QueryBuilderTest extends SimpleTestSuite { ) ) - // val order = - // orderBy(c.name.asc) - - val q = Select(proj, tables, cond) - println(q) + val q = Select(proj, tables, cond).orderBy(c.name.desc) + q match { + case Select.Ordered(Select.SimpleSelect(proj, from, where, group), sb, vempty) => + assert(vempty.isEmpty) + assertEquals(sb, OrderBy(SelectExpr.SelectColumn(c.name), OrderBy.OrderType.Desc)) + assertEquals(11, proj.size) + from match { + case FromExpr.From(_) => + fail("Unexpected from value") + case FromExpr.Joined(f, joins) => + assertEquals(f, FromExpr.From(c)) + assertEquals(2, joins.size) + joins.head match { + case Join.InnerJoin(tbl, cond) => + assertEquals(tbl, owner) + assertEquals(cond, c.ownerId === owner.id) + case _ => + fail("Unexpected join result") + } + joins.tail.head match { + case Join.LeftJoin(tbl, cond) => + assertEquals(tbl, lecturer) + assertEquals(cond, c.lecturerId === lecturer.id) + case _ => + fail("Unexpected join result") + } + } + assertEquals(group, None) + assert(where.isDefined) + case _ => + fail("Unexpected case") + } } } diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala index 9b38d94f..a5b22f81 100644 --- a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala @@ -4,7 +4,6 @@ import minitest._ import docspell.store.qb._ import docspell.store.qb.model._ import docspell.store.qb.DSL._ -import docspell.common._ object DoobieQueryTest extends SimpleTestSuite { @@ -23,28 +22,11 @@ object DoobieQueryTest extends SimpleTestSuite { ) val q = Select(proj, table, cond) - val frag = DoobieQuery.select(q) + val frag = DoobieQuery(q) assertEquals( frag.toString, """Fragment("SELECT c.id, c.name, c.owner_id, c.lecturer_id, c.lessons FROM course c INNER JOIN person o ON c.owner_id = o.id LEFT JOIN person l ON c.lecturer_id = l.id WHERE (LOWER(c.name) LIKE ? AND o.name = ? )")""" ) } - test("basic update") { - val p = PersonRecord.table - - val update = PersonRecord.update(p.name.set("john"), p.id.set(15L)).where(p.id >= 2) - - println(DoobieQuery.update(update)) - - } - - test("basic insert") { - val p = PersonRecord(1, "John", Timestamp.Epoch) - - val insert = PersonRecord.insertAll(p) - - println(insert) - - } } diff --git a/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala index 867a41de..2024fd1f 100644 --- a/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala +++ b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala @@ -28,7 +28,4 @@ object CourseRecord { def as(alias: String): Table = Table(Some(alias)) - def update: UpdateTable = - UpdateTable(Table(None), None, Seq.empty) - } diff --git a/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala index bfcee143..a328c6a8 100644 --- a/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala +++ b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala @@ -2,9 +2,6 @@ package docspell.store.qb.model import docspell.store.qb._ import docspell.common._ -import doobie.implicits._ -import docspell.store.impl.DoobieMeta._ -import doobie._ case class PersonRecord(id: Long, name: String, created: Timestamp) @@ -24,15 +21,4 @@ object PersonRecord { def as(alias: String): Table = Table(Some(alias)) - def table: Table = Table(None) - - def update(set: UpdateTable.Setter[_], sets: UpdateTable.Setter[_]*): UpdateTable = - UpdateTable(table, None, sets :+ set) - - def insertAll(v: PersonRecord): ConnectionIO[Int] = - InsertTable( - table, - table.all, - fr"${v.id},${v.name},${v.created}" - ).toFragment.update.run } From c5c7f7ed3bf7f55b0a7088211b3b01b9224d2717 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 8 Dec 2020 22:43:17 +0100 Subject: [PATCH 04/38] Convert equipment record --- .../scala/docspell/store/impl/Implicits.scala | 8 +- .../main/scala/docspell/store/qb/DSL.scala | 42 ++++++++- .../scala/docspell/store/queries/QItem.scala | 49 +++++----- .../docspell/store/records/REquipment.scala | 89 +++++++++++-------- 4 files changed, 127 insertions(+), 61 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala index 9e18b3e1..edafa832 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala @@ -1,3 +1,9 @@ package docspell.store.impl -object Implicits extends DoobieMeta with DoobieSyntax +object Implicits extends DoobieMeta with DoobieSyntax { + + implicit final class LegacySyntax(col: docspell.store.qb.Column[_]) { + def oldColumn: Column = + Column(col.name) + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 63eb7083..4844b662 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -26,10 +26,28 @@ trait DSL extends DoobieMeta { DBFunction.Count(c, "cn") def and(c: Condition, cs: Condition*): Condition = - Condition.And(c, cs.toVector) + c match { + case Condition.And(head, tail) => + Condition.And(head, tail ++ (c +: cs.toVector)) + case _ => + Condition.And(c, cs.toVector) + } def or(c: Condition, cs: Condition*): Condition = - Condition.Or(c, cs.toVector) + c match { + case Condition.Or(head, tail) => + Condition.Or(head, tail ++ (c +: cs.toVector)) + case _ => + Condition.Or(c, cs.toVector) + } + + def not(c: Condition): Condition = + c match { + case Condition.Not(el) => + el + case _ => + Condition.Not(c) + } def where(c: Condition, cs: Condition*): Condition = and(c, cs: _*) @@ -71,7 +89,27 @@ trait DSL extends DoobieMeta { def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) + } + implicit final class ConditionOps(c: Condition) { + + def &&(other: Condition): Condition = + and(c, other) + + def &&?(other: Option[Condition]): Condition = + other.map(ce => &&(ce)).getOrElse(c) + + def ||(other: Condition): Condition = + or(c, other) + + def ||?(other: Option[Condition]): Condition = + other.map(ce => ||(ce)).getOrElse(c) + + def negate: Condition = + not(c) + + def unary_! : Condition = + not(c) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 494bf2b2..2207a29f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -87,13 +87,14 @@ object QItem { } def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val OC = ROrganization.Columns.all.map(_.prefix("o")) - val P0C = RPerson.Columns.all.map(_.prefix("p0")) - val P1C = RPerson.Columns.all.map(_.prefix("p1")) - val EC = REquipment.Columns.all.map(_.prefix("e")) - val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) - val FC = List(RFolder.Columns.id, RFolder.Columns.name).map(_.prefix("f")) + val equip = REquipment.as("e") + val IC = RItem.Columns.all.map(_.prefix("i")) + val OC = ROrganization.Columns.all.map(_.prefix("o")) + val P0C = RPerson.Columns.all.map(_.prefix("p0")) + val P1C = RPerson.Columns.all.map(_.prefix("p1")) + val EC = equip.all.map(_.oldColumn).map(_.prefix("e")) + val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) + val FC = List(RFolder.Columns.id, RFolder.Columns.name).map(_.prefix("f")) val cq = selectSimple( @@ -110,9 +111,11 @@ object QItem { fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson .prefix("i") .is(RPerson.Columns.pid.prefix("p1")) ++ - fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment + fr"LEFT JOIN" ++ Fragment.const( + equip.tableName + ) ++ fr"e ON" ++ RItem.Columns.concEquipment .prefix("i") - .is(REquipment.Columns.eid.prefix("e")) ++ + .is(equip.eid.oldColumn.prefix("e")) ++ fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo .prefix("i") .is(RItem.Columns.id.prefix("ref")) ++ @@ -305,16 +308,16 @@ object QItem { moreCols: Seq[Fragment], ctes: (String, Fragment)* ): Fragment = { + val equip = REquipment.as("e1") val IC = RItem.Columns val AC = RAttachment.Columns val PC = RPerson.Columns val OC = ROrganization.Columns - val EC = REquipment.Columns val FC = RFolder.Columns val itemCols = IC.all val personCols = List(PC.pid, PC.name) val orgCols = List(OC.oid, OC.name) - val equipCols = List(EC.eid, EC.name) + val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) val folderCols = List(FC.id, FC.name) val cvItem = RCustomFieldValue.Columns.itemId.prefix("cv") @@ -335,8 +338,8 @@ object QItem { PC.name.prefix("p0").f, PC.pid.prefix("p1").f, PC.name.prefix("p1").f, - EC.eid.prefix("e1").f, - EC.name.prefix("e1").f, + equip.eid.oldColumn.prefix("e1").f, + equip.name.oldColumn.prefix("e1").f, FC.id.prefix("f1").f, FC.name.prefix("f1").f, // sql uses 1 for first character @@ -357,7 +360,11 @@ object QItem { val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid.is(q.account.collective)) val withEquips = - selectSimple(equipCols, REquipment.table, EC.cid.is(q.account.collective)) + selectSimple( + equipCols, + Fragment.const(equip.tableName), + equip.cid.oldColumn.is(q.account.collective) + ) val withFolder = selectSimple(folderCols, RFolder.table, FC.collective.is(q.account.collective)) val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ @@ -384,7 +391,7 @@ object QItem { fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment .prefix("i") - .is(EC.eid.prefix("e1")) ++ + .is(equip.eid.oldColumn.prefix("e1")) ++ fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1")) ++ (if (q.customValues.isEmpty) Fragment.empty else @@ -396,10 +403,10 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - val IC = RItem.Columns - val PC = RPerson.Columns - val OC = ROrganization.Columns - val EC = REquipment.Columns + val equip = REquipment.as("e1") + val IC = RItem.Columns + val PC = RPerson.Columns + val OC = ROrganization.Columns // inclusive tags are AND-ed val tagSelectsIncl = q.tagsInclude @@ -432,7 +439,7 @@ object QItem { OC.name.prefix("o0").lowerLike(n), PC.name.prefix("p0").lowerLike(n), PC.name.prefix("p1").lowerLike(n), - EC.name.prefix("e1").lowerLike(n), + equip.name.oldColumn.prefix("e1").lowerLike(n), IC.name.prefix("i").lowerLike(n), IC.notes.prefix("i").lowerLike(n) ) @@ -441,7 +448,7 @@ object QItem { RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson), ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg), RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson), - REquipment.Columns.eid.prefix("e1").isOrDiscard(q.concEquip), + equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip), RFolder.Columns.id.prefix("f1").isOrDiscard(q.folder), if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty else diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala index 3a7f6d2f..c7542e41 100644 --- a/modules/store/src/main/scala/docspell/store/records/REquipment.scala +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -1,8 +1,8 @@ package docspell.store.records import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -16,70 +16,85 @@ case class REquipment( ) {} object REquipment { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "equipment" - val table = fr"equipment" - - object Columns { - val eid = Column("eid") - val cid = Column("cid") - val name = Column("name") - val created = Column("created") - val updated = Column("updated") + val eid = Column[Ident]("eid", this) + val cid = Column[Ident]("cid", this) + val name = Column[String]("name", this) + val created = Column[Timestamp]("created", this) + val updated = Column[Timestamp]("updated", this) val all = List(eid, cid, name, created, updated) } - import Columns._ + + def as(alias: String): Table = + Table(Some(alias)) def insert(v: REquipment): ConnectionIO[Int] = { - val sql = - insertRow(table, all, fr"${v.eid},${v.cid},${v.name},${v.created},${v.updated}") - sql.update.run + val t = Table(None) + DML + .insert( + t, + t.all, + fr"${v.eid},${v.cid},${v.name},${v.created},${v.updated}" + ) + .update + .run } def update(v: REquipment): ConnectionIO[Int] = { - def sql(now: Timestamp) = - updateRow( - table, - and(eid.is(v.eid), cid.is(v.cid)), - commas( - cid.setTo(v.cid), - name.setTo(v.name), - updated.setTo(now) - ) - ) + val t = Table(None) for { now <- Timestamp.current[ConnectionIO] - n <- sql(now).update.run + n <- DML + .update( + t, + where(t.eid === v.eid, t.cid === v.cid), + DML.set( + t.cid.setTo(v.cid), + t.name.setTo(v.name), + t.updated.setTo(now) + ) + ) } yield n } def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = { - val sql = selectCount(eid, table, and(cid.is(coll), name.is(ename))) + val t = Table(None) + val sql = run(select(count(t.eid)), from(t), where(t.cid === coll, t.name === ename)) sql.query[Int].unique.map(_ > 0) } def findById(id: Ident): ConnectionIO[Option[REquipment]] = { - val sql = selectSimple(all, table, eid.is(id)) + val t = Table(None) + val sql = run(select(t.all), from(t), t.eid === id) sql.query[REquipment].option } def findAll( coll: Ident, nameQ: Option[String], - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[REquipment]] = { - val q = Seq(cid.is(coll)) ++ (nameQ match { - case Some(str) => Seq(name.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) - val sql = selectSimple(all, table, and(q)) ++ orderBy(order(Columns).f) + val t = Table(None) + + val q = t.cid === coll &&? nameQ + .map(str => s"%${str.toLowerCase}%") + .map(v => t.name.like(v)) + + val sql = Select(select(t.all), from(t), q).orderBy(order(t)).run sql.query[REquipment].to[Vector] } - def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = - selectSimple(List(eid, name), table, and(cid.is(coll), name.lowerLike(equipName))) + def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = { + val t = Table(None) + run(select(List(t.eid, t.name)), from(t), t.cid === coll && t.name.like(equipName)) .query[IdRef] .to[Vector] + } - def delete(id: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(eid.is(id), cid.is(coll))).update.run + def delete(id: Ident, coll: Ident): ConnectionIO[Int] = { + val t = Table(None) + DML.delete(t, t.eid === id && t.cid === coll).update.run + } } From 10b49fccf87800775bc4fa40789edebe3cc555ad Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 9 Dec 2020 00:00:10 +0100 Subject: [PATCH 05/38] Converting user and userimap records --- .../docspell/joex/analysis/RegexNerFile.scala | 7 +- .../scala/docspell/store/impl/Implicits.scala | 11 ++ .../main/scala/docspell/store/qb/Column.scala | 5 +- .../scala/docspell/store/qb/Condition.scala | 2 + .../main/scala/docspell/store/qb/DSL.scala | 11 ++ .../store/qb/impl/ConditionBuilder.scala | 4 + .../docspell/store/queries/QFolder.scala | 29 ++-- .../scala/docspell/store/queries/QLogin.scala | 12 +- .../scala/docspell/store/queries/QMails.scala | 19 +-- .../scala/docspell/store/records/RUser.scala | 115 +++++++------- .../docspell/store/records/RUserEmail.scala | 25 +++- .../docspell/store/records/RUserImap.scala | 140 ++++++++++-------- 12 files changed, 229 insertions(+), 151 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 6cff49f7..7187e147 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -144,6 +144,7 @@ object RegexNerFile { def max(col: Column, table: Fragment, cidCol: Column): Fragment = selectSimple(col.max ++ fr"as t", table, cidCol.is(collective)) + val equip = REquipment.as("e") val sql = List( max( @@ -152,7 +153,11 @@ object RegexNerFile { ROrganization.Columns.cid ), max(RPerson.Columns.updated, RPerson.table, RPerson.Columns.cid), - max(REquipment.Columns.updated, REquipment.table, REquipment.Columns.cid) + max( + equip.updated.oldColumn, + Fragment.const(equip.tableName), + equip.cid.oldColumn + ) ) .reduce(_ ++ fr"UNION ALL" ++ _) diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala index edafa832..30cba7ca 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala @@ -5,5 +5,16 @@ object Implicits extends DoobieMeta with DoobieSyntax { implicit final class LegacySyntax(col: docspell.store.qb.Column[_]) { def oldColumn: Column = Column(col.name) + + def column: Column = { + val c = col.alias match { + case Some(a) => oldColumn.as(a) + case None => oldColumn + } + col.table.alias match { + case Some(p) => c.prefix(p) + case None => c + } + } } } diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala index 3f3fa1ab..90936f90 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Column.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -1,5 +1,8 @@ package docspell.store.qb -case class Column[A](name: String, table: TableDef, alias: Option[String] = None) +case class Column[A](name: String, table: TableDef, alias: Option[String] = None) { + def as(alias: String): Column[A] = + copy(alias = Some(alias)) +} object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 45f9a4c7..a4414f31 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -13,6 +13,8 @@ object Condition { case class CompareCol[A](col1: Column[A], op: Operator, col2: Column[A]) extends Condition + case class InSubSelect[A](col: Column[A], subSelect: Select) extends Condition + case class And(c: Condition, cs: Vector[Condition]) extends Condition case class Or(c: Condition, cs: Vector[Condition]) extends Condition case class Not(c: Condition) extends Condition diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 4844b662..76b52342 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -72,9 +72,17 @@ trait DSL extends DoobieMeta { def ===(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Eq, value) + //TODO find some better way around the cast + def ====(value: String): Condition = + Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.Eq, value) + def like(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.LowerLike, value) + //TODO find some better way around the cast + def likes(value: String): Condition = + Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.LowerLike, value) + def <=(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Lte, value) @@ -87,6 +95,9 @@ trait DSL extends DoobieMeta { def <(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Lt, value) + def in(subsel: Select): Condition = + Condition.InSubSelect(col, subsel) + def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index 5a37733f..a7c8f536 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -33,6 +33,10 @@ object ConditionBuilder { } c1Frag ++ operator(op) ++ c2Frag + case Condition.InSubSelect(col, subsel) => + val sub = DoobieQuery(subsel) + SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ sql")" + case Condition.And(c, cs) => val inner = cs.prepended(c).map(build).reduce(_ ++ and ++ _) if (cs.isEmpty) inner diff --git a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala index 9c922d48..2f71fe0d 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala @@ -136,15 +136,16 @@ object QFolder { } def findById(id: Ident, account: AccountId): ConnectionIO[Option[FolderDetail]] = { + val user = RUser.as("u") val mUserId = RFolderMember.Columns.user.prefix("m") val mFolderId = RFolderMember.Columns.folder.prefix("m") - val uId = RUser.Columns.uid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") + val uId = user.uid.column + val uLogin = user.login.column val sColl = RFolder.Columns.collective.prefix("s") val sId = RFolder.Columns.id.prefix("s") val from = RFolderMember.table ++ fr"m INNER JOIN" ++ - RUser.table ++ fr"u ON" ++ mUserId.is(uId) ++ fr"INNER JOIN" ++ + Fragment.const(user.tableName) ++ fr"u ON" ++ mUserId.is(uId) ++ fr"INNER JOIN" ++ RFolder.table ++ fr"s ON" ++ mFolderId.is(sId) val memberQ = selectSimple( @@ -187,8 +188,9 @@ object QFolder { // inner join user_ u on u.uid = s.owner // where s.cid = 'eike'; - val uId = RUser.Columns.uid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") + val user = RUser.as("u") + val uId = user.uid.column + val uLogin = user.login.column val sId = RFolder.Columns.id.prefix("s") val sOwner = RFolder.Columns.owner.prefix("s") val sName = RFolder.Columns.name.prefix("s") @@ -199,11 +201,11 @@ object QFolder { //CTE val cte: Fragment = { val from1 = RFolderMember.table ++ fr"m INNER JOIN" ++ - RUser.table ++ fr"u ON" ++ uId.is(mUser) ++ fr"INNER JOIN" ++ + Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(mUser) ++ fr"INNER JOIN" ++ RFolder.table ++ fr"s ON" ++ sId.is(mFolder) val from2 = RFolder.table ++ fr"s INNER JOIN" ++ - RUser.table ++ fr"u ON" ++ uId.is(sOwner) + Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(sOwner) withCTE( "memberlogin" -> @@ -232,7 +234,7 @@ object QFolder { ) val from = RFolder.table ++ fr"s INNER JOIN" ++ - RUser.table ++ fr"u ON" ++ uId.is(sOwner) + Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(sOwner) val where = sColl.is(account.collective) :: idQ.toList @@ -247,17 +249,20 @@ object QFolder { /** Select all folder_id where the given account is member or owner. */ def findMemberFolderIds(account: AccountId): Fragment = { + val user = RUser.as("u") val fId = RFolder.Columns.id.prefix("f") val fOwner = RFolder.Columns.owner.prefix("f") val fColl = RFolder.Columns.collective.prefix("f") - val uId = RUser.Columns.uid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") + val uId = user.uid.column + val uLogin = user.login.column val mFolder = RFolderMember.Columns.folder.prefix("m") val mUser = RFolderMember.Columns.user.prefix("m") selectSimple( Seq(fId), - RFolder.table ++ fr"f INNER JOIN" ++ RUser.table ++ fr"u ON" ++ fOwner.is(uId), + RFolder.table ++ fr"f INNER JOIN" ++ Fragment.const( + user.tableName + ) ++ fr"u ON" ++ fOwner.is(uId), and(fColl.is(account.collective), uLogin.is(account.user)) ) ++ fr"UNION ALL" ++ @@ -266,7 +271,7 @@ object QFolder { RFolderMember.table ++ fr"m INNER JOIN" ++ RFolder.table ++ fr"f ON" ++ fId.is( mFolder ) ++ - fr"INNER JOIN" ++ RUser.table ++ fr"u ON" ++ uId.is(mUser), + fr"INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(mUser), and(fColl.is(account.collective), uLogin.is(account.user)) ) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala index 4554772d..7dfdf59f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala @@ -5,7 +5,6 @@ import cats.data.OptionT import docspell.common._ import docspell.store.impl.Implicits._ import docspell.store.records.RCollective.{Columns => CC} -import docspell.store.records.RUser.{Columns => UC} import docspell.store.records.{RCollective, RRememberMe, RUser} import doobie._ @@ -23,16 +22,17 @@ object QLogin { ) def findUser(acc: AccountId): ConnectionIO[Option[Data]] = { - val ucid = UC.cid.prefix("u") - val login = UC.login.prefix("u") - val pass = UC.password.prefix("u") - val ustate = UC.state.prefix("u") + val user = RUser.as("u") + val ucid = user.cid.column + val login = user.login.column + val pass = user.password.column + val ustate = user.state.column val cstate = CC.state.prefix("c") val ccid = CC.id.prefix("c") val sql = selectSimple( List(ucid, login, pass, cstate, ustate), - RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c", + Fragment.const(user.tableName) ++ fr"u, " ++ RCollective.table ++ fr"c", and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective)) ) diff --git a/modules/store/src/main/scala/docspell/store/queries/QMails.scala b/modules/store/src/main/scala/docspell/store/queries/QMails.scala index 90046d33..08330362 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QMails.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QMails.scala @@ -45,19 +45,20 @@ object QMails { } private def partialFind: (Seq[Column], Fragment) = { - val iId = RItem.Columns.id.prefix("i") - val tItem = RSentMailItem.Columns.itemId.prefix("t") - val tMail = RSentMailItem.Columns.sentMailId.prefix("t") - val mId = RSentMail.Columns.id.prefix("m") - val mUser = RSentMail.Columns.uid.prefix("m") - val uId = RUser.Columns.uid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") + val user = RUser.as("u") + val iId = RItem.Columns.id.prefix("i") + val tItem = RSentMailItem.Columns.itemId.prefix("t") + val tMail = RSentMailItem.Columns.sentMailId.prefix("t") + val mId = RSentMail.Columns.id.prefix("m") + val mUser = RSentMail.Columns.uid.prefix("m") - val cols = RSentMail.Columns.all.map(_.prefix("m")) :+ uLogin + val cols = RSentMail.Columns.all.map(_.prefix("m")) :+ user.login.column val from = RSentMail.table ++ fr"m INNER JOIN" ++ RSentMailItem.table ++ fr"t ON" ++ tMail.is(mId) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ tItem.is(iId) ++ - fr"INNER JOIN" ++ RUser.table ++ fr"u ON" ++ uId.is(mUser) + fr"INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ user.uid.column.is( + mUser + ) (cols, from) } diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index a304cabf..5d21d08a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -1,8 +1,8 @@ package docspell.store.records import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -20,86 +20,99 @@ case class RUser( ) {} object RUser { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "user_" - val table = fr"user_" - - object Columns { - val uid = Column("uid") - val cid = Column("cid") - val login = Column("login") - val password = Column("password") - val state = Column("state") - val email = Column("email") - val loginCount = Column("logincount") - val lastLogin = Column("lastlogin") - val created = Column("created") + val uid = Column[Ident]("uid", this) + val login = Column[Ident]("login", this) + val cid = Column[Ident]("cid", this) + val password = Column[Password]("password", this) + val state = Column[UserState]("state", this) + val email = Column[String]("email", this) + val loginCount = Column[Int]("logincount", this) + val lastLogin = Column[Timestamp]("lastlogin", this) + val created = Column[Timestamp]("created", this) val all = List(uid, login, cid, password, state, email, loginCount, lastLogin, created) } - import Columns._ + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RUser): ConnectionIO[Int] = { - val sql = insertRow( - table, - Columns.all, + val t = Table(None) + val sql = DML.insert( + t, + t.all, fr"${v.uid},${v.login},${v.cid},${v.password},${v.state},${v.email},${v.loginCount},${v.lastLogin},${v.created}" ) sql.update.run } def update(v: RUser): ConnectionIO[Int] = { - val sql = updateRow( - table, - and(login.is(v.login), cid.is(v.cid)), - commas( - state.setTo(v.state), - email.setTo(v.email), - loginCount.setTo(v.loginCount), - lastLogin.setTo(v.lastLogin) + val t = Table(None) + DML.update( + t, + t.login === v.login && t.cid === v.cid, + DML.set( + t.state.setTo(v.state), + t.email.setTo(v.email), + t.loginCount.setTo(v.loginCount), + t.lastLogin.setTo(v.lastLogin) ) ) - sql.update.run } - def exists(loginName: Ident): ConnectionIO[Boolean] = - selectCount(uid, table, login.is(loginName)).query[Int].unique.map(_ > 0) + def exists(loginName: Ident): ConnectionIO[Boolean] = { + val t = Table(None) + run(select(count(t.uid)), from(t), t.login === loginName).query[Int].unique.map(_ > 0) + } def findByAccount(aid: AccountId): ConnectionIO[Option[RUser]] = { - val sql = selectSimple(all, table, and(cid.is(aid.collective), login.is(aid.user))) + val t = Table(None) + val sql = + run(select(t.all), from(t), t.cid === aid.collective && t.login === aid.user) sql.query[RUser].option } def findById(userId: Ident): ConnectionIO[Option[RUser]] = { - val sql = selectSimple(all, table, uid.is(userId)) + val t = Table(None) + val sql = run(select(t.all), from(t), t.uid === userId) sql.query[RUser].option } - def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RUser]] = { - val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) + def findAll(coll: Ident, order: Table => Column[_]): ConnectionIO[Vector[RUser]] = { + val t = Table(None) + val sql = Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).run sql.query[RUser].to[Vector] } - def updateLogin(accountId: AccountId): ConnectionIO[Int] = - currentTime.flatMap(t => - updateRow( - table, - and(cid.is(accountId.collective), login.is(accountId.user)), - commas( - loginCount.f ++ fr"=" ++ loginCount.f ++ fr"+ 1", - lastLogin.setTo(t) + def updateLogin(accountId: AccountId): ConnectionIO[Int] = { + val t = Table(None) + def stmt(now: Timestamp) = + DML.update( + t, + t.cid === accountId.collective && t.login === accountId.user, + DML.set( + t.loginCount.increment(1), + t.lastLogin.setTo(now) ) - ).update.run + ) + Timestamp.current[ConnectionIO].flatMap(stmt) + } + + def updatePassword(accountId: AccountId, hashedPass: Password): ConnectionIO[Int] = { + val t = Table(None) + DML.update( + t, + t.cid === accountId.collective && t.login === accountId.user, + DML.set(t.password.setTo(hashedPass)) ) + } - def updatePassword(accountId: AccountId, hashedPass: Password): ConnectionIO[Int] = - updateRow( - table, - and(cid.is(accountId.collective), login.is(accountId.user)), - password.setTo(hashedPass) - ).update.run - - def delete(user: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(cid.is(coll), login.is(user))).update.run + def delete(user: Ident, coll: Ident): ConnectionIO[Int] = { + val t = Table(None) + DML.delete(t, t.cid === coll && t.login === user).update.run + } } diff --git a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala index 270df3c0..5c7b2802 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala @@ -168,12 +168,16 @@ object RUserEmail { nameQ: Option[String], exact: Boolean ): Query0[RUserEmail] = { + val user = RUser.as("u") val mUid = uid.prefix("m") val mName = name.prefix("m") - val uId = RUser.Columns.uid.prefix("u") - val uColl = RUser.Columns.cid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") - val from = table ++ fr"m INNER JOIN" ++ RUser.table ++ fr"u ON" ++ mUid.is(uId) + val uId = user.uid.column + val uColl = user.cid.column + val uLogin = user.login.column + val from = + table ++ fr"m INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ mUid.is( + uId + ) val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) ++ (nameQ match { case Some(str) if exact => Seq(mName.is(str)) case Some(str) => Seq(mName.lowerLike(s"%${str.toLowerCase}%")) @@ -194,14 +198,19 @@ object RUserEmail { findByAccount0(accId, Some(name.id), true).option def delete(accId: AccountId, connName: Ident): ConnectionIO[Int] = { - val uId = RUser.Columns.uid - val uColl = RUser.Columns.cid - val uLogin = RUser.Columns.login + val user = RUser.as("u") + val uId = user.uid.column + val uColl = user.cid.column + val uLogin = user.login.column val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) deleteFrom( table, - fr"uid in (" ++ selectSimple(Seq(uId), RUser.table, and(cond)) ++ fr") AND" ++ name + fr"uid in (" ++ selectSimple( + Seq(uId), + Fragment.const(user.tableName), + and(cond) + ) ++ fr") AND" ++ name .is( connName ) diff --git a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala index 4babbe76..5cff7c83 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala @@ -5,8 +5,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -92,19 +92,19 @@ object RUserImap { now ) - val table = fr"userimap" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "userimap" - object Columns { - val id = Column("id") - val uid = Column("uid") - val name = Column("name") - val imapHost = Column("imap_host") - val imapPort = Column("imap_port") - val imapUser = Column("imap_user") - val imapPass = Column("imap_password") - val imapSsl = Column("imap_ssl") - val imapCertCheck = Column("imap_certcheck") - val created = Column("created") + val id = Column[Ident]("id", this) + val uid = Column[Ident]("uid", this) + val name = Column[Ident]("name", this) + val imapHost = Column[String]("imap_host", this) + val imapPort = Column[Int]("imap_port", this) + val imapUser = Column[String]("imap_user", this) + val imapPass = Column[Password]("imap_password", this) + val imapSsl = Column[SSLType]("imap_ssl", this) + val imapCertCheck = Column[Boolean]("imap_certcheck", this) + val created = Column[Timestamp]("created", this) val all = List( id, @@ -120,52 +120,64 @@ object RUserImap { ) } - import Columns._ + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RUserImap): ConnectionIO[Int] = - insertRow( - table, - all, - sql"${v.id},${v.uid},${v.name},${v.imapHost},${v.imapPort},${v.imapUser},${v.imapPassword},${v.imapSsl},${v.imapCertCheck},${v.created}" - ).update.run - - def update(eId: Ident, v: RUserImap): ConnectionIO[Int] = - updateRow( - table, - id.is(eId), - commas( - name.setTo(v.name), - imapHost.setTo(v.imapHost), - imapPort.setTo(v.imapPort), - imapUser.setTo(v.imapUser), - imapPass.setTo(v.imapPassword), - imapSsl.setTo(v.imapSsl), - imapCertCheck.setTo(v.imapCertCheck) + def insert(v: RUserImap): ConnectionIO[Int] = { + val t = Table(None) + DML + .insert( + t, + t.all, + sql"${v.id},${v.uid},${v.name},${v.imapHost},${v.imapPort},${v.imapUser},${v.imapPassword},${v.imapSsl},${v.imapCertCheck},${v.created}" ) - ).update.run + .update + .run + } - def findByUser(userId: Ident): ConnectionIO[Vector[RUserImap]] = - selectSimple(all, table, uid.is(userId)).query[RUserImap].to[Vector] + def update(eId: Ident, v: RUserImap): ConnectionIO[Int] = { + val t = Table(None) + DML.update( + t, + t.id === eId, + DML.set( + t.name.setTo(v.name), + t.imapHost.setTo(v.imapHost), + t.imapPort.setTo(v.imapPort), + t.imapUser.setTo(v.imapUser), + t.imapPass.setTo(v.imapPassword), + t.imapSsl.setTo(v.imapSsl), + t.imapCertCheck.setTo(v.imapCertCheck) + ) + ) + } + + def findByUser(userId: Ident): ConnectionIO[Vector[RUserImap]] = { + val t = Table(None) + run(select(t.all), from(t), t.uid === userId).query[RUserImap].to[Vector] + } private def findByAccount0( accId: AccountId, nameQ: Option[String], exact: Boolean ): Query0[RUserImap] = { - val mUid = uid.prefix("m") - val mName = name.prefix("m") - val uId = RUser.Columns.uid.prefix("u") - val uColl = RUser.Columns.cid.prefix("u") - val uLogin = RUser.Columns.login.prefix("u") - val from = table ++ fr"m INNER JOIN" ++ RUser.table ++ fr"u ON" ++ mUid.is(uId) - val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) ++ (nameQ match { - case Some(str) if exact => Seq(mName.is(str)) - case Some(str) => Seq(mName.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) + val m = RUserImap.as("m") + val u = RUser.as("u") - (selectSimple(all.map(_.prefix("m")), from, and(cond)) ++ orderBy(mName.f)) - .query[RUserImap] + val nameFilter = + nameQ.map { str => + if (exact) m.name ==== str + else m.name.likes(s"%${str.toLowerCase}%") + } + + val sql = Select( + select(m.all), + from(m).innerJoin(u, m.uid === u.uid), + u.cid === accId.collective && u.login === accId.user &&? nameFilter + ).orderBy(m.name).run + + sql.query[RUserImap] } def findByAccount( @@ -178,26 +190,28 @@ object RUserImap { findByAccount0(accId, Some(name.id), true).option def delete(accId: AccountId, connName: Ident): ConnectionIO[Int] = { - val uId = RUser.Columns.uid - val uColl = RUser.Columns.cid - val uLogin = RUser.Columns.login - val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) + val t = Table(None) + val u = RUser.as("u") + val subsel = + Select(select(u.uid), from(u), u.cid === accId.collective && u.login === accId.user) - deleteFrom( - table, - fr"uid in (" ++ selectSimple(Seq(uId), RUser.table, and(cond)) ++ fr") AND" ++ name - .is( - connName - ) - ).update.run + DML + .delete( + t, + t.uid.in(subsel) && t.name === connName + ) + .update + .run } def exists(accId: AccountId, name: Ident): ConnectionIO[Boolean] = getByName(accId, name).map(_.isDefined) - def exists(userId: Ident, connName: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(uid.is(userId), name.is(connName))) + def exists(userId: Ident, connName: Ident): ConnectionIO[Boolean] = { + val t = Table(None) + run(select(count(t.id)), from(t), t.uid === userId && t.name === connName) .query[Int] .unique .map(_ > 0) + } } From 5cbf0d56028f24b0aa5db3724063bd63fad3d523 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 9 Dec 2020 23:18:09 +0100 Subject: [PATCH 06/38] Convert more records --- .../scala/docspell/store/qb/Condition.scala | 5 + .../main/scala/docspell/store/qb/DML.scala | 29 +++- .../main/scala/docspell/store/qb/DSL.scala | 11 +- .../scala/docspell/store/qb/GroupBy.scala | 10 ++ .../main/scala/docspell/store/qb/Select.scala | 5 +- .../store/qb/impl/ConditionBuilder.scala | 7 + .../docspell/store/queries/QCollective.scala | 47 ++++-- .../scala/docspell/store/queries/QItem.scala | 42 ++--- .../docspell/store/records/REquipment.scala | 4 +- .../scala/docspell/store/records/RNode.scala | 73 +++++---- .../docspell/store/records/RSource.scala | 19 +-- .../scala/docspell/store/records/RTag.scala | 150 +++++++++--------- .../docspell/store/records/RTagItem.scala | 55 ++++--- .../docspell/store/records/RTagSource.scala | 39 ++--- .../scala/docspell/store/records/RUser.scala | 5 +- .../docspell/store/records/RUserEmail.scala | 146 +++++++++-------- .../docspell/store/records/RUserImap.scala | 13 +- .../docspell/store/records/TagItemName.scala | 35 ++-- 18 files changed, 381 insertions(+), 314 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index a4414f31..88a5487f 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -1,5 +1,7 @@ package docspell.store.qb +import cats.data.NonEmptyList + import doobie._ sealed trait Condition {} @@ -14,6 +16,9 @@ object Condition { extends Condition case class InSubSelect[A](col: Column[A], subSelect: Select) extends Condition + case class InValues[A](col: Column[A], values: NonEmptyList[A], lower: Boolean)(implicit + val P: Put[A] + ) extends Condition case class And(c: Condition, cs: Vector[Condition]) extends Condition case class Or(c: Condition, cs: Vector[Condition]) extends Condition diff --git a/modules/store/src/main/scala/docspell/store/qb/DML.scala b/modules/store/src/main/scala/docspell/store/qb/DML.scala index dbbe1c79..194ee3db 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DML.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DML.scala @@ -8,25 +8,42 @@ import doobie.implicits._ object DML { private val comma = fr"," - def delete(table: TableDef, cond: Condition): Fragment = + def delete(table: TableDef, cond: Condition): ConnectionIO[Int] = + deleteFragment(table, cond).update.run + + def deleteFragment(table: TableDef, cond: Condition): Fragment = fr"DELETE FROM" ++ FromExprBuilder.buildTable(table) ++ fr"WHERE" ++ ConditionBuilder .build(cond) - def insert(table: TableDef, cols: Seq[Column[_]], values: Fragment): Fragment = + def insert(table: TableDef, cols: Seq[Column[_]], values: Fragment): ConnectionIO[Int] = + insertFragment(table, cols, List(values)).update.run + + def insertMany( + table: TableDef, + cols: Seq[Column[_]], + values: Seq[Fragment] + ): ConnectionIO[Int] = + insertFragment(table, cols, values).update.run + + def insertFragment( + table: TableDef, + cols: Seq[Column[_]], + values: Seq[Fragment] + ): Fragment = fr"INSERT INTO" ++ FromExprBuilder.buildTable(table) ++ sql"(" ++ cols .map(SelectExprBuilder.columnNoPrefix) - .reduce(_ ++ comma ++ _) ++ fr") VALUES (" ++ - values ++ fr")" + .reduce(_ ++ comma ++ _) ++ fr") VALUES" ++ + values.map(f => sql"(" ++ f ++ sql")").reduce(_ ++ comma ++ _) def update( table: TableDef, cond: Condition, setter: Seq[Setter[_]] ): ConnectionIO[Int] = - update(table, Some(cond), setter).update.run + updateFragment(table, Some(cond), setter).update.run - def update( + def updateFragment( table: TableDef, cond: Option[Condition], setter: Seq[Setter[_]] diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 76b52342..0b7b62a2 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -1,5 +1,7 @@ package docspell.store.qb +import cats.data.NonEmptyList + import docspell.store.impl.DoobieMeta import docspell.store.qb.impl.DoobieQuery @@ -50,7 +52,8 @@ trait DSL extends DoobieMeta { } def where(c: Condition, cs: Condition*): Condition = - and(c, cs: _*) + if (cs.isEmpty) c + else and(c, cs: _*) implicit final class ColumnOps[A](col: Column[A]) { @@ -98,6 +101,12 @@ trait DSL extends DoobieMeta { def in(subsel: Select): Condition = Condition.InSubSelect(col, subsel) + def in(values: NonEmptyList[A])(implicit P: Put[A]): Condition = + Condition.InValues(col, values, false) + + def inLower(values: NonEmptyList[A])(implicit P: Put[A]): Condition = + Condition.InValues(col, values, true) + def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) } diff --git a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala index a12e6448..51250513 100644 --- a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala +++ b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala @@ -1,3 +1,13 @@ package docspell.store.qb case class GroupBy(name: SelectExpr, names: Vector[SelectExpr], having: Option[Condition]) + +object GroupBy { + + def apply(c: Column[_], cs: Column[_]*): GroupBy = + GroupBy( + SelectExpr.SelectColumn(c), + cs.toVector.map(SelectExpr.SelectColumn.apply), + None + ) +} diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index accb90a0..fd660332 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -38,7 +38,10 @@ object Select { from: FromExpr, where: Option[Condition], groupBy: Option[GroupBy] - ) extends Select + ) extends Select { + def group(gb: GroupBy): SimpleSelect = + copy(groupBy = Some(gb)) + } case class Union(q: Select, qs: Vector[Select]) extends Select diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index a7c8f536..1f99df1e 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -8,6 +8,7 @@ import _root_.doobie.{Query => _, _} object ConditionBuilder { val or = fr"OR" val and = fr"AND" + val comma = fr"," val parenOpen = Fragment.const0("(") val parenClose = Fragment.const0(")") @@ -37,6 +38,12 @@ object ConditionBuilder { val sub = DoobieQuery(subsel) SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ sql")" + case c @ Condition.InValues(col, values, toLower) => + val cfrag = if (toLower) lower(col) else SelectExprBuilder.column(col) + cfrag ++ sql" IN (" ++ values.toList + .map(a => buildValue(a)(c.P)) + .reduce(_ ++ comma ++ _) ++ sql")" + case Condition.And(c, cs) => val inner = cs.prepended(c).map(build).reduce(_ ++ and ++ _) if (cs.isEmpty) inner diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index a1d162af..a180623f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -6,6 +6,7 @@ import fs2.Stream import docspell.common.ContactKind import docspell.common.{Direction, Ident} import docspell.store.impl.Implicits._ +import docspell.store.qb.{GroupBy, Select} import docspell.store.records._ import doobie._ @@ -77,25 +78,39 @@ object QCollective { } def tagCloud(coll: Ident): ConnectionIO[List[TagCount]] = { - val TC = RTag.Columns - val RC = RTagItem.Columns + import docspell.store.qb.DSL._ - val q3 = fr"SELECT" ++ commas( - TC.all.map(_.prefix("t").f) ++ Seq(fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") - ) ++ - fr"FROM" ++ RTagItem.table ++ fr"r" ++ - fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId - .prefix("r") - .is(TC.tid.prefix("t")) ++ - fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ - fr"GROUP BY" ++ commas( - TC.name.prefix("t").f, - TC.tid.prefix("t").f, - TC.category.prefix("t").f - ) + val ti = RTagItem.as("ti") + val t = RTag.as("t") + val sql = + Select( + select(t.all) ++ select(count(ti.itemId)), + from(ti).innerJoin(t, ti.tagId === t.tid), + t.cid === coll + ).group(GroupBy(t.name, t.tid, t.category)) - q3.query[TagCount].to[List] + sql.run.query[TagCount].to[List] } +// def tagCloud2(coll: Ident): ConnectionIO[List[TagCount]] = { +// val tagItem = RTagItem.as("r") +// val TC = RTag.Columns +// +// val q3 = fr"SELECT" ++ commas( +// TC.all.map(_.prefix("t").f) ++ Seq(fr"count(" ++ tagItem.itemId.column.f ++ fr")") +// ) ++ +// fr"FROM" ++ Fragment.const(tagItem.tableName) ++ fr"r" ++ +// fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ tagItem.tagId.column +// .prefix("r") +// .is(TC.tid.prefix("t")) ++ +// fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ +// fr"GROUP BY" ++ commas( +// TC.name.prefix("t").f, +// TC.tid.prefix("t").f, +// TC.category.prefix("t").f +// ) +// +// q3.query[TagCount].to[List] +// } def getContacts( coll: Ident, diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 2207a29f..29d6ea89 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -412,9 +412,9 @@ object QItem { val tagSelectsIncl = q.tagsInclude .map(tid => selectSimple( - List(RTagItem.Columns.itemId), - RTagItem.table, - RTagItem.Columns.tagId.is(tid) + List(RTagItem.t.itemId.column), + Fragment.const(RTagItem.t.tableName), + RTagItem.t.tagId.column.is(tid) ) ) ++ q.tagCategoryIncl.map(cat => TagItemName.itemsInCategory(NonEmptyList.of(cat))) @@ -755,33 +755,35 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mText = RAttachmentMeta.Columns.content.prefix("m") - val tiItem = RTagItem.Columns.itemId.prefix("ti") - val tiTag = RTagItem.Columns.tagId.prefix("ti") - val tId = RTag.Columns.tid.prefix("t") - val tName = RTag.Columns.name.prefix("t") - val tCat = RTag.Columns.category.prefix("t") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") + val aId = RAttachment.Columns.id.prefix("a") + val aItem = RAttachment.Columns.itemId.prefix("a") + val mId = RAttachmentMeta.Columns.id.prefix("m") + val mText = RAttachmentMeta.Columns.content.prefix("m") + val tagItem = RTagItem.as("ti") //Columns.itemId.prefix("ti") + //val tiTag = RTagItem.Columns.tagId.prefix("ti") + val tag = RTag.as("t") +// val tId = RTag.Columns.tid.prefix("t") +// val tName = RTag.Columns.name.prefix("t") +// val tCat = RTag.Columns.category.prefix("t") + val iId = RItem.Columns.id.prefix("i") + val iColl = RItem.Columns.cid.prefix("i") val cte = withCTE( "tags" -> selectSimple( - Seq(tiItem, tId, tName), - RTagItem.table ++ fr"ti INNER JOIN" ++ - RTag.table ++ fr"t ON" ++ tId.is(tiTag), - and(tiItem.is(itemId), tCat.is(tagCategory)) + Seq(tagItem.itemId.column, tag.tid.column, tag.name.column), + Fragment.const(RTagItem.t.tableName) ++ fr"ti INNER JOIN" ++ + Fragment.const(tag.tableName) ++ fr"t ON" ++ tag.tid.column + .is(tagItem.tagId.column), + and(tagItem.itemId.column.is(itemId), tag.category.column.is(tagCategory)) ) ) - val cols = Seq(mText, tId, tName) + val cols = Seq(mText, tag.tid.column, tag.name.column) val from = RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"LEFT JOIN" ++ - fr"tags t ON" ++ RTagItem.Columns.itemId.prefix("t").is(iId) + fr"tags t ON" ++ RTagItem.t.itemId.oldColumn.prefix("t").is(iId) val where = and( diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala index c7542e41..7883731e 100644 --- a/modules/store/src/main/scala/docspell/store/records/REquipment.scala +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -38,8 +38,6 @@ object REquipment { t.all, fr"${v.eid},${v.cid},${v.name},${v.created},${v.updated}" ) - .update - .run } def update(v: REquipment): ConnectionIO[Int] = { @@ -95,6 +93,6 @@ object REquipment { def delete(id: Ident, coll: Ident): ConnectionIO[Int] = { val t = Table(None) - DML.delete(t, t.eid === id && t.cid === coll).update.run + DML.delete(t, t.eid === id && t.cid === coll) } } diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index 5afa5944..8bc7bff1 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -4,8 +4,8 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -23,35 +23,42 @@ object RNode { def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] = Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now)) - val table = fr"node" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "node" - object Columns { - val id = Column("id") - val nodeType = Column("type") - val url = Column("url") - val updated = Column("updated") - val created = Column("created") + val id = Column[Ident]("id", this) + val nodeType = Column[NodeType]("type", this) + val url = Column[LenientUri]("url", this) + val updated = Column[Timestamp]("updated", this) + val created = Column[Timestamp]("created", this) val all = List(id, nodeType, url, updated, created) } - import Columns._ - def insert(v: RNode): ConnectionIO[Int] = - insertRow( - table, - all, + def as(alias: String): Table = + Table(Some(alias)) + + def insert(v: RNode): ConnectionIO[Int] = { + val t = Table(None) + DML.insert( + t, + t.all, fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}" - ).update.run + ) + } - def update(v: RNode): ConnectionIO[Int] = - updateRow( - table, - id.is(v.id), - commas( - nodeType.setTo(v.nodeType), - url.setTo(v.url), - updated.setTo(v.updated) + def update(v: RNode): ConnectionIO[Int] = { + val t = Table(None) + DML + .update( + t, + t.id === v.id, + DML.set( + t.nodeType.setTo(v.nodeType), + t.url.setTo(v.url), + t.updated.setTo(v.updated) + ) ) - ).update.run + } def set(v: RNode): ConnectionIO[Int] = for { @@ -59,12 +66,18 @@ object RNode { k <- if (n == 0) insert(v) else 0.pure[ConnectionIO] } yield n + k - def delete(appId: Ident): ConnectionIO[Int] = - (fr"DELETE FROM" ++ table ++ where(id.is(appId))).update.run + def delete(appId: Ident): ConnectionIO[Int] = { + val t = Table(None) + DML.delete(t, t.id === appId) + } - def findAll(nt: NodeType): ConnectionIO[Vector[RNode]] = - selectSimple(all, table, nodeType.is(nt)).query[RNode].to[Vector] + def findAll(nt: NodeType): ConnectionIO[Vector[RNode]] = { + val t = Table(None) + run(select(t.all), from(t), t.nodeType === nt).query[RNode].to[Vector] + } - def findById(nodeId: Ident): ConnectionIO[Option[RNode]] = - selectSimple(all, table, id.is(nodeId)).query[RNode].option + def findById(nodeId: Ident): ConnectionIO[Option[RNode]] = { + val t = Table(None) + run(select(t.all), from(t), t.id === nodeId).query[RNode].option + } } diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala index 3e126df0..11b87270 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -60,14 +60,12 @@ object RSource { val table = Table(None) - def insert(v: RSource): ConnectionIO[Int] = { - val sql = DML.insert( + def insert(v: RSource): ConnectionIO[Int] = + DML.insert( table, table.all, fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}" ) - sql.update.run - } def updateNoCounter(v: RSource): ConnectionIO[Int] = DML.update( @@ -85,12 +83,11 @@ object RSource { ) def incrementCounter(source: String, coll: Ident): ConnectionIO[Int] = - DML - .update( - table, - where(table.abbrev === source, table.cid === coll), - DML.set(table.counter.increment(1)) - ) + DML.update( + table, + where(table.abbrev === source, table.cid === coll), + DML.set(table.counter.increment(1)) + ) def existsById(id: Ident): ConnectionIO[Boolean] = { val sql = run(select(count(table.sid)), from(table), where(table.sid === id)) @@ -130,7 +127,7 @@ object RSource { } def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = - DML.delete(table, where(table.sid === sourceId, table.cid === coll)).update.run + DML.delete(table, where(table.sid === sourceId, table.cid === coll)) def removeFolder(folderId: Ident): ConnectionIO[Int] = { val empty: Option[Ident] = None diff --git a/modules/store/src/main/scala/docspell/store/records/RTag.scala b/modules/store/src/main/scala/docspell/store/records/RTag.scala index cc206d39..3285fbc4 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTag.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala @@ -4,8 +4,8 @@ import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -19,101 +19,97 @@ case class RTag( ) {} object RTag { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "tag" - val table = fr"tag" - - object Columns { - val tid = Column("tid") - val cid = Column("cid") - val name = Column("name") - val category = Column("category") - val created = Column("created") + val tid = Column[Ident]("tid", this) + val cid = Column[Ident]("cid", this) + val name = Column[String]("name", this) + val category = Column[String]("category", this) + val created = Column[Timestamp]("created", this) val all = List(tid, cid, name, category, created) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RTag): ConnectionIO[Int] = { - val sql = - insertRow( - table, - all, - fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}" - ) - sql.update.run - } + def insert(v: RTag): ConnectionIO[Int] = + DML.insert( + T, + T.all, + fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}" + ) - def update(v: RTag): ConnectionIO[Int] = { - val sql = updateRow( - table, - and(tid.is(v.tagId), cid.is(v.collective)), - commas( - cid.setTo(v.collective), - name.setTo(v.name), - category.setTo(v.category) + def update(v: RTag): ConnectionIO[Int] = + DML.update( + T, + T.tid === v.tagId && T.cid === v.collective, + DML.set( + T.cid.setTo(v.collective), + T.name.setTo(v.name), + T.category.setTo(v.category) ) ) - sql.update.run - } def findById(id: Ident): ConnectionIO[Option[RTag]] = { - val sql = selectSimple(all, table, tid.is(id)) + val sql = run(select(T.all), from(T), T.tid === id) sql.query[RTag].option } def findByIdAndCollective(id: Ident, coll: Ident): ConnectionIO[Option[RTag]] = { - val sql = selectSimple(all, table, and(tid.is(id), cid.is(coll))) + val sql = run(select(T.all), from(T), T.tid === id && T.cid === coll) sql.query[RTag].option } def existsByName(tag: RTag): ConnectionIO[Boolean] = { - val sql = selectCount( - tid, - table, - and(cid.is(tag.collective), name.is(tag.name)) - ) + val sql = + run(select(count(T.tid)), from(T), T.cid === tag.collective && T.name === tag.name) sql.query[Int].unique.map(_ > 0) } def findAll( coll: Ident, nameQ: Option[String], - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[RTag]] = { - val q = Seq(cid.is(coll)) ++ (nameQ match { - case Some(str) => Seq(name.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) - val sql = selectSimple(all, table, and(q)) ++ orderBy(order(Columns).f) - sql.query[RTag].to[Vector] + val nameFilter = nameQ.map(s => T.name.like(s"%${s.toLowerCase}%")) + val sql = + Select(select(T.all), from(T), T.cid === coll &&? nameFilter).orderBy(order(T)) + sql.run.query[RTag].to[Vector] } def findAllById(ids: List[Ident]): ConnectionIO[Vector[RTag]] = - selectSimple(all, table, tid.isIn(ids.map(id => sql"$id").toSeq)) - .query[RTag] - .to[Vector] + NonEmptyList.fromList(ids) match { + case Some(nel) => + run(select(T.all), from(T), T.tid.in(nel)) + .query[RTag] + .to[Vector] + case None => + Vector.empty.pure[ConnectionIO] + } def findByItem(itemId: Ident): ConnectionIO[Vector[RTag]] = { - val rcol = all.map(_.prefix("t")) - (selectSimple( - rcol, - table ++ fr"t," ++ RTagItem.table ++ fr"i", - and( - RTagItem.Columns.itemId.prefix("i").is(itemId), - RTagItem.Columns.tagId.prefix("i").is(tid.prefix("t")) - ) - ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector] + val ti = RTagItem.as("i") + val t = RTag.as("t") + val sql = + Select( + select(t.all), + from(t).innerJoin(ti, ti.tagId === t.tid), + ti.itemId === itemId + ).orderBy(t.name.asc) + sql.run.query[RTag].to[Vector] } def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = { - val rcol = all.map(_.prefix("t")) - (selectSimple( - rcol, - table ++ fr"t," ++ RTagSource.table ++ fr"s", - and( - RTagSource.Columns.sourceId.prefix("s").is(source), - RTagSource.Columns.tagId.prefix("s").is(tid.prefix("t")) - ) - ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector] + val s = RTagSource.as("s") + val t = RTag.as("t") + val sql = + Select( + select(t.all), + from(t).innerJoin(s, s.tagId === t.tid), + s.sourceId === source + ).orderBy(t.name.asc) + sql.run.query[RTag].to[Vector] } def findAllByNameOrId( @@ -121,16 +117,22 @@ object RTag { coll: Ident ): ConnectionIO[Vector[RTag]] = { val idList = - NonEmptyList.fromList(nameOrIds.flatMap(s => Ident.fromString(s).toOption)).toSeq - val nameList = NonEmptyList.fromList(nameOrIds.map(_.toLowerCase)).toSeq - - val cond = idList.flatMap(ids => Seq(tid.isIn(ids))) ++ - nameList.flatMap(ns => Seq(name.isLowerIn(ns))) - - if (cond.isEmpty) Vector.empty.pure[ConnectionIO] - else selectSimple(all, table, and(cid.is(coll), or(cond))).query[RTag].to[Vector] + NonEmptyList.fromList(nameOrIds.flatMap(s => Ident.fromString(s).toOption)) + val nameList = NonEmptyList.fromList(nameOrIds.map(_.toLowerCase)) + (idList, nameList) match { + case (Some(ids), _) => + val cond = + T.cid === coll && (T.tid.in(ids) ||? nameList.map(names => T.name.in(names))) + run(select(T.all), from(T), cond).query[RTag].to[Vector] + case (_, Some(names)) => + val cond = + T.cid === coll && (T.name.in(names) ||? idList.map(ids => T.tid.in(ids))) + run(select(T.all), from(T), cond).query[RTag].to[Vector] + case (None, None) => + Vector.empty.pure[ConnectionIO] + } } def delete(tagId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(tid.is(tagId), cid.is(coll))).update.run + DML.delete(T, T.tid === tagId && T.cid === coll) } diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index c9aad9db..b702beb3 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -4,8 +4,8 @@ import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -13,41 +13,45 @@ import doobie.implicits._ case class RTagItem(tagItemId: Ident, itemId: Ident, tagId: Ident) {} object RTagItem { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "tagitem" - val table = fr"tagitem" - - object Columns { - val tagItemId = Column("tagitemid") - val itemId = Column("itemid") - val tagId = Column("tid") + val tagItemId = Column[Ident]("tagitemid", this) + val itemId = Column[Ident]("itemid", this) + val tagId = Column[Ident]("tid", this) val all = List(tagItemId, itemId, tagId) } - import Columns._ + val t = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RTagItem): ConnectionIO[Int] = - insertRow(table, all, fr"${v.tagItemId},${v.itemId},${v.tagId}").update.run + DML.insert(t, t.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") def deleteItemTags(item: Ident): ConnectionIO[Int] = - deleteFrom(table, itemId.is(item)).update.run + DML.delete(t, t.itemId === item) def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] = { - val itemsFiltered = - RItem.filterItemsFragment(items, cid) - val sql = fr"DELETE FROM" ++ table ++ fr"WHERE" ++ itemId.isIn(itemsFiltered) - - sql.update.run + print(cid) + DML.delete(t, t.itemId.in(items)) + //TODO match those of the collective + //val itemsFiltered = + // RItem.filterItemsFragment(items, cid) + //val sql = fr"DELETE FROM" ++ Fragment.const(t.tableName) ++ fr"WHERE" ++ + // t.itemId.isIn(itemsFiltered) + //sql.update.run } def deleteTag(tid: Ident): ConnectionIO[Int] = - deleteFrom(table, tagId.is(tid)).update.run + DML.delete(t, t.tagId === tid) def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] = - selectSimple(all, table, itemId.is(item)).query[RTagItem].to[Vector] + run(select(t.all), from(t), t.itemId === item).query[RTagItem].to[Vector] def findAllIn(item: Ident, tags: Seq[Ident]): ConnectionIO[Vector[RTagItem]] = NonEmptyList.fromList(tags.toList) match { case Some(nel) => - selectSimple(all, table, and(itemId.is(item), tagId.isIn(nel))) + run(select(t.all), from(t), t.itemId === item && t.tagId.in(nel)) .query[RTagItem] .to[Vector] case None => @@ -59,7 +63,7 @@ object RTagItem { case None => 0.pure[ConnectionIO] case Some(nel) => - deleteFrom(table, and(itemId.is(item), tagId.isIn(nel))).update.run + DML.delete(t, t.itemId === item && t.tagId.in(nel)) } def setAllTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = @@ -69,11 +73,12 @@ object RTagItem { entities <- tags.toList.traverse(tagId => Ident.randomId[ConnectionIO].map(id => RTagItem(id, item, tagId)) ) - n <- insertRows( - table, - all, - entities.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") - ).update.run + n <- DML + .insertMany( + t, + t.all, + entities.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") + ) } yield n def appendTags(item: Ident, tags: List[Ident]): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RTagSource.scala b/modules/store/src/main/scala/docspell/store/records/RTagSource.scala index f94cc88e..8558c98a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagSource.scala @@ -4,8 +4,8 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -13,31 +13,33 @@ import doobie.implicits._ case class RTagSource(id: Ident, sourceId: Ident, tagId: Ident) {} object RTagSource { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "tagsource" - val table = fr"tagsource" - - object Columns { - val id = Column("id") - val sourceId = Column("source_id") - val tagId = Column("tag_id") + val id = Column[Ident]("id", this) + val sourceId = Column[Ident]("source_id", this) + val tagId = Column[Ident]("tag_id", this) val all = List(id, sourceId, tagId) } - import Columns._ + + private val t = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def createNew[F[_]: Sync](source: Ident, tag: Ident): F[RTagSource] = Ident.randomId[F].map(id => RTagSource(id, source, tag)) def insert(v: RTagSource): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.sourceId},${v.tagId}").update.run + DML.insert(t, t.all, fr"${v.id},${v.sourceId},${v.tagId}") def deleteSourceTags(source: Ident): ConnectionIO[Int] = - deleteFrom(table, sourceId.is(source)).update.run + DML.delete(t, t.sourceId === source) def deleteTag(tid: Ident): ConnectionIO[Int] = - deleteFrom(table, tagId.is(tid)).update.run + DML.delete(t, t.tagId === tid) def findBySource(source: Ident): ConnectionIO[Vector[RTagSource]] = - selectSimple(all, table, sourceId.is(source)).query[RTagSource].to[Vector] + run(select(t.all), from(t), t.sourceId === source).query[RTagSource].to[Vector] def setAllTags(source: Ident, tags: Seq[Ident]): ConnectionIO[Int] = if (tags.isEmpty) 0.pure[ConnectionIO] @@ -46,11 +48,12 @@ object RTagSource { entities <- tags.toList.traverse(tagId => Ident.randomId[ConnectionIO].map(id => RTagSource(id, source, tagId)) ) - n <- insertRows( - table, - all, - entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}") - ).update.run + n <- DML + .insertMany( + t, + t.all, + entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}") + ) } yield n } diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index 5d21d08a..2862f729 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -42,12 +42,11 @@ object RUser { def insert(v: RUser): ConnectionIO[Int] = { val t = Table(None) - val sql = DML.insert( + DML.insert( t, t.all, fr"${v.uid},${v.login},${v.cid},${v.password},${v.state},${v.email},${v.loginCount},${v.lastLogin},${v.created}" ) - sql.update.run } def update(v: RUser): ConnectionIO[Int] = { @@ -113,6 +112,6 @@ object RUser { def delete(user: Ident, coll: Ident): ConnectionIO[Int] = { val t = Table(None) - DML.delete(t, t.cid === coll && t.login === user).update.run + DML.delete(t, t.cid === coll && t.login === user) } } diff --git a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala index 5c7b2802..17935f08 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala @@ -5,8 +5,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -101,22 +101,22 @@ object RUserEmail { mailReplyTo, now ) + final case class Table(alias: Option[String]) extends TableDef { - val table = fr"useremail" + val tableName = "useremail" - object Columns { - val id = Column("id") - val uid = Column("uid") - val name = Column("name") - val smtpHost = Column("smtp_host") - val smtpPort = Column("smtp_port") - val smtpUser = Column("smtp_user") - val smtpPass = Column("smtp_password") - val smtpSsl = Column("smtp_ssl") - val smtpCertCheck = Column("smtp_certcheck") - val mailFrom = Column("mail_from") - val mailReplyTo = Column("mail_replyto") - val created = Column("created") + val id = Column[Ident]("id", this) + val uid = Column[Ident]("uid", this) + val name = Column[Ident]("name", this) + val smtpHost = Column[String]("smtp_host", this) + val smtpPort = Column[Int]("smtp_port", this) + val smtpUser = Column[String]("smtp_user", this) + val smtpPass = Column[Password]("smtp_password", this) + val smtpSsl = Column[SSLType]("smtp_ssl", this) + val smtpCertCheck = Column[Boolean]("smtp_certcheck", this) + val mailFrom = Column[MailAddress]("mail_from", this) + val mailReplyTo = Column[MailAddress]("mail_replyto", this) + val created = Column[Timestamp]("created", this) val all = List( id, @@ -134,58 +134,61 @@ object RUserEmail { ) } - import Columns._ + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RUserEmail): ConnectionIO[Int] = - insertRow( - table, - all, + def insert(v: RUserEmail): ConnectionIO[Int] = { + val t = Table(None) + DML.insert( + t, + t.all, sql"${v.id},${v.uid},${v.name},${v.smtpHost},${v.smtpPort},${v.smtpUser},${v.smtpPassword},${v.smtpSsl},${v.smtpCertCheck},${v.mailFrom},${v.mailReplyTo},${v.created}" - ).update.run + ) + } - def update(eId: Ident, v: RUserEmail): ConnectionIO[Int] = - updateRow( - table, - id.is(eId), - commas( - name.setTo(v.name), - smtpHost.setTo(v.smtpHost), - smtpPort.setTo(v.smtpPort), - smtpUser.setTo(v.smtpUser), - smtpPass.setTo(v.smtpPassword), - smtpSsl.setTo(v.smtpSsl), - smtpCertCheck.setTo(v.smtpCertCheck), - mailFrom.setTo(v.mailFrom), - mailReplyTo.setTo(v.mailReplyTo) + def update(eId: Ident, v: RUserEmail): ConnectionIO[Int] = { + val t = Table(None) + DML.update( + t, + t.id === eId, + DML.set( + t.name.setTo(v.name), + t.smtpHost.setTo(v.smtpHost), + t.smtpPort.setTo(v.smtpPort), + t.smtpUser.setTo(v.smtpUser), + t.smtpPass.setTo(v.smtpPassword), + t.smtpSsl.setTo(v.smtpSsl), + t.smtpCertCheck.setTo(v.smtpCertCheck), + t.mailFrom.setTo(v.mailFrom), + t.mailReplyTo.setTo(v.mailReplyTo) ) - ).update.run + ) + } - def findByUser(userId: Ident): ConnectionIO[Vector[RUserEmail]] = - selectSimple(all, table, uid.is(userId)).query[RUserEmail].to[Vector] + def findByUser(userId: Ident): ConnectionIO[Vector[RUserEmail]] = { + val t = Table(None) + run(select(t.all), from(t), t.uid === userId).query[RUserEmail].to[Vector] + } private def findByAccount0( accId: AccountId, nameQ: Option[String], exact: Boolean ): Query0[RUserEmail] = { - val user = RUser.as("u") - val mUid = uid.prefix("m") - val mName = name.prefix("m") - val uId = user.uid.column - val uColl = user.cid.column - val uLogin = user.login.column - val from = - table ++ fr"m INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ mUid.is( - uId - ) - val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) ++ (nameQ match { - case Some(str) if exact => Seq(mName.is(str)) - case Some(str) => Seq(mName.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) + val user = RUser.as("u") + val email = as("m") - (selectSimple(all.map(_.prefix("m")), from, and(cond)) ++ orderBy(mName.f)) - .query[RUserEmail] + val nameFilter = nameQ.map(s => + if (exact) email.name ==== s else email.name.likes(s"%${s.toLowerCase}%") + ) + + val sql = Select( + select(email.all), + from(email).innerJoin(user, email.uid === user.uid), + user.cid === accId.collective && user.login === accId.user &&? nameFilter + ).orderBy(email.name) + + sql.run.query[RUserEmail] } def findByAccount( @@ -198,31 +201,26 @@ object RUserEmail { findByAccount0(accId, Some(name.id), true).option def delete(accId: AccountId, connName: Ident): ConnectionIO[Int] = { - val user = RUser.as("u") - val uId = user.uid.column - val uColl = user.cid.column - val uLogin = user.login.column - val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) + val user = RUser.as("u") - deleteFrom( - table, - fr"uid in (" ++ selectSimple( - Seq(uId), - Fragment.const(user.tableName), - and(cond) - ) ++ fr") AND" ++ name - .is( - connName - ) - ).update.run + val subsel = Select( + select(user.uid), + from(user), + user.cid === accId.collective && user.login === accId.user + ) + + val t = Table(None) + DML.delete(t, t.uid.in(subsel) && t.name === connName) } def exists(accId: AccountId, name: Ident): ConnectionIO[Boolean] = getByName(accId, name).map(_.isDefined) - def exists(userId: Ident, connName: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(uid.is(userId), name.is(connName))) + def exists(userId: Ident, connName: Ident): ConnectionIO[Boolean] = { + val t = Table(None) + run(select(count(t.id)), from(t), t.uid === userId && t.name === connName) .query[Int] .unique .map(_ > 0) + } } diff --git a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala index 5cff7c83..d40b9c03 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala @@ -131,8 +131,6 @@ object RUserImap { t.all, sql"${v.id},${v.uid},${v.name},${v.imapHost},${v.imapPort},${v.imapUser},${v.imapPassword},${v.imapSsl},${v.imapCertCheck},${v.created}" ) - .update - .run } def update(eId: Ident, v: RUserImap): ConnectionIO[Int] = { @@ -195,13 +193,10 @@ object RUserImap { val subsel = Select(select(u.uid), from(u), u.cid === accId.collective && u.login === accId.user) - DML - .delete( - t, - t.uid.in(subsel) && t.name === connName - ) - .update - .run + DML.delete( + t, + t.uid.in(subsel) && t.name === connName + ) } def exists(accId: AccountId, name: Ident): ConnectionIO[Boolean] = diff --git a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala index fffa1f61..45ef618a 100644 --- a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala +++ b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala @@ -3,10 +3,9 @@ package docspell.store.records import cats.data.NonEmptyList import docspell.common._ -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ import doobie._ -import doobie.implicits._ /** A helper class combining information from `RTag` and `RTagItem`. * This is not a "record", there is no corresponding table. @@ -24,37 +23,27 @@ object TagItemName { def itemsInCategory(cats: NonEmptyList[String]): Fragment = { val catsLower = cats.map(_.toLowerCase) - val tiItem = RTagItem.Columns.itemId.prefix("ti") - val tiTag = RTagItem.Columns.tagId.prefix("ti") - val tCat = RTag.Columns.category.prefix("t") - val tId = RTag.Columns.tid.prefix("t") - - val from = RTag.table ++ fr"t INNER JOIN" ++ - RTagItem.table ++ fr"ti ON" ++ tiTag.is(tId) - + val ti = RTagItem.as("ti") + val t = RTag.as("t") + val join = from(t).innerJoin(ti, t.tid === ti.tagId) if (cats.tail.isEmpty) - selectSimple(List(tiItem), from, tCat.lowerIs(catsLower.head)) + run(select(ti.itemId), join, t.category.likes(catsLower.head)) else - selectSimple(List(tiItem), from, tCat.isLowerIn(catsLower)) + run(select(ti.itemId), join, t.category.inLower(cats)) } def itemsWithTagOrCategory(tags: List[Ident], cats: List[String]): Fragment = { val catsLower = cats.map(_.toLowerCase) - val tiItem = RTagItem.Columns.itemId.prefix("ti") - val tiTag = RTagItem.Columns.tagId.prefix("ti") - val tCat = RTag.Columns.category.prefix("t") - val tId = RTag.Columns.tid.prefix("t") - - val from = RTag.table ++ fr"t INNER JOIN" ++ - RTagItem.table ++ fr"ti ON" ++ tiTag.is(tId) - + val ti = RTagItem.as("ti") + val t = RTag.as("t") + val join = from(t).innerJoin(ti, t.tid === ti.tagId) (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match { case (Some(tagNel), Some(catNel)) => - selectSimple(List(tiItem), from, or(tId.isIn(tagNel), tCat.isLowerIn(catNel))) + run(select(ti.itemId), join, t.tid.in(tagNel) || t.category.inLower(catNel)) case (Some(tagNel), None) => - selectSimple(List(tiItem), from, tId.isIn(tagNel)) + run(select(ti.itemId), join, t.tid.in(tagNel)) case (None, Some(catNel)) => - selectSimple(List(tiItem), from, tCat.isLowerIn(catNel)) + run(select(ti.itemId), join, t.category.inLower(catNel)) case (None, None) => Fragment.empty } From fe4815c73715844f64e6c0b2bed5e8e281c1a314 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 10 Dec 2020 21:12:00 +0100 Subject: [PATCH 07/38] Convert RSentMail --- .../scala/docspell/store/queries/QMails.scala | 31 +++++++------ .../docspell/store/records/RSentMail.scala | 45 ++++++++++--------- .../store/records/RSentMailItem.scala | 34 +++++++------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/queries/QMails.scala b/modules/store/src/main/scala/docspell/store/queries/QMails.scala index 08330362..06b528fa 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QMails.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QMails.scala @@ -21,7 +21,8 @@ object QMails { def findMail(coll: Ident, mailId: Ident): ConnectionIO[Option[(RSentMail, Ident)]] = { val iColl = RItem.Columns.cid.prefix("i") - val mId = RSentMail.Columns.id.prefix("m") + val smail = RSentMail.as("m") + val mId = smail.id.column val (cols, from) = partialFind @@ -31,9 +32,11 @@ object QMails { } def findMails(coll: Ident, itemId: Ident): ConnectionIO[Vector[(RSentMail, Ident)]] = { - val iColl = RItem.Columns.cid.prefix("i") - val tItem = RSentMailItem.Columns.itemId.prefix("t") - val mCreated = RSentMail.Columns.created.prefix("m") + val smailitem = RSentMailItem.as("t") + val smail = RSentMail.as("m") + val iColl = RItem.Columns.cid.prefix("i") + val tItem = smailitem.itemId.column + val mCreated = smail.created.column val (cols, from) = partialFind @@ -45,16 +48,18 @@ object QMails { } private def partialFind: (Seq[Column], Fragment) = { - val user = RUser.as("u") - val iId = RItem.Columns.id.prefix("i") - val tItem = RSentMailItem.Columns.itemId.prefix("t") - val tMail = RSentMailItem.Columns.sentMailId.prefix("t") - val mId = RSentMail.Columns.id.prefix("m") - val mUser = RSentMail.Columns.uid.prefix("m") + val user = RUser.as("u") + val smailitem = RSentMailItem.as("t") + val smail = RSentMail.as("m") + val iId = RItem.Columns.id.prefix("i") + val tItem = smailitem.itemId.column + val tMail = smailitem.sentMailId.column + val mId = smail.id.column + val mUser = smail.uid.column - val cols = RSentMail.Columns.all.map(_.prefix("m")) :+ user.login.column - val from = RSentMail.table ++ fr"m INNER JOIN" ++ - RSentMailItem.table ++ fr"t ON" ++ tMail.is(mId) ++ + val cols = smail.all.map(_.column) :+ user.login.column + val from = Fragment.const(smail.tableName) ++ fr"m INNER JOIN" ++ + Fragment.const(smailitem.tableName) ++ fr"t ON" ++ tMail.is(mId) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ tItem.is(iId) ++ fr"INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ user.uid.column.is( mUser diff --git a/modules/store/src/main/scala/docspell/store/records/RSentMail.scala b/modules/store/src/main/scala/docspell/store/records/RSentMail.scala index 2a35f92b..cd4aa224 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSentMail.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSentMail.scala @@ -7,8 +7,8 @@ import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -78,18 +78,19 @@ object RSentMail { si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created))) } yield (sm, si) - val table = fr"sentmail" + final case class Table(alias: Option[String]) extends TableDef { - object Columns { - val id = Column("id") - val uid = Column("uid") - val messageId = Column("message_id") - val sender = Column("sender") - val connName = Column("conn_name") - val subject = Column("subject") - val recipients = Column("recipients") - val body = Column("body") - val created = Column("created") + val tableName = "sentmail" + + val id = Column[Ident]("id", this) + val uid = Column[Ident]("uid", this) + val messageId = Column[String]("message_id", this) + val sender = Column[MailAddress]("sender", this) + val connName = Column[Ident]("conn_name", this) + val subject = Column[String]("subject", this) + val recipients = Column[List[MailAddress]]("recipients", this) + val body = Column[String]("body", this) + val created = Column[Timestamp]("created", this) val all = List( id, @@ -104,27 +105,29 @@ object RSentMail { ) } - import Columns._ + private val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RSentMail): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, sql"${v.id},${v.uid},${v.messageId},${v.sender},${v.connName},${v.subject},${v.recipients},${v.body},${v.created}" - ).update.run + ) def findByUser(userId: Ident): Stream[ConnectionIO, RSentMail] = - selectSimple(all, table, uid.is(userId)).query[RSentMail].stream + run(select(T.all), from(T), T.uid === userId).query[RSentMail].stream def delete(mailId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(mailId)).update.run + DML.delete(T, T.id === mailId) def deleteByItem(item: Ident): ConnectionIO[Int] = for { list <- RSentMailItem.findSentMailIdsByItem(item) n1 <- RSentMailItem.deleteAllByItem(item) n0 <- NonEmptyList.fromList(list.toList) match { - case Some(nel) => deleteFrom(table, id.isIn(nel)).update.run + case Some(nel) => DML.delete(T, T.id.in(nel)) case None => 0.pure[ConnectionIO] } } yield n0 + n1 diff --git a/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala b/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala index d17e29ec..04dffc0b 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala @@ -4,8 +4,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -29,13 +29,13 @@ object RSentMailItem { now <- created.map(_.pure[F]).getOrElse(Timestamp.current[F]) } yield RSentMailItem(id, itemId, sentmailId, now) - val table = fr"sentmailitem" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "sentmailitem" - object Columns { - val id = Column("id") - val itemId = Column("item_id") - val sentMailId = Column("sentmail_id") - val created = Column("created") + val id = Column[Ident]("id", this) + val itemId = Column[Ident]("item_id", this) + val sentMailId = Column[Ident]("sentmail_id", this) + val created = Column[Timestamp]("created", this) val all = List( id, @@ -45,21 +45,23 @@ object RSentMailItem { ) } - import Columns._ + private val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RSentMailItem): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, sql"${v.id},${v.itemId},${v.sentMailId},${v.created}" - ).update.run + ) def deleteMail(mailId: Ident): ConnectionIO[Int] = - deleteFrom(table, sentMailId.is(mailId)).update.run + DML.delete(T, T.sentMailId === mailId) def findSentMailIdsByItem(item: Ident): ConnectionIO[Set[Ident]] = - selectSimple(Seq(sentMailId), table, itemId.is(item)).query[Ident].to[Set] + run(select(Seq(T.sentMailId)), from(T), T.itemId === item).query[Ident].to[Set] def deleteAllByItem(item: Ident): ConnectionIO[Int] = - deleteFrom(table, itemId.is(item)).update.run + DML.delete(T, T.itemId === item) } From 3cef932ccd7e334278e7f2dab683301ae94b4ad6 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 11 Dec 2020 01:05:54 +0100 Subject: [PATCH 08/38] Convert more records --- .../docspell/joex/analysis/RegexNerFile.scala | 33 ++-- .../scala/docspell/store/qb/DBFunction.scala | 5 + .../main/scala/docspell/store/qb/DSL.scala | 21 ++- .../scala/docspell/store/qb/FromExpr.scala | 10 +- .../scala/docspell/store/qb/TableDef.scala | 9 ++ .../store/qb/impl/FromExprBuilder.scala | 3 + .../store/qb/impl/SelectExprBuilder.scala | 13 +- .../docspell/store/queries/QCollective.scala | 61 ++----- .../scala/docspell/store/queries/QItem.scala | 93 ++++++----- .../store/queries/QOrganization.scala | 150 +++++++----------- .../docspell/store/records/RContact.scala | 66 ++++---- .../docspell/store/records/REquipment.scala | 1 + .../store/records/ROrganization.scala | 118 +++++++------- .../docspell/store/records/RPerson.scala | 145 +++++++++-------- .../docspell/store/records/RRememberMe.scala | 42 ++--- 15 files changed, 376 insertions(+), 394 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 7187e147..fb5f097c 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -136,32 +136,21 @@ object RegexNerFile { object Sql { import doobie._ - import doobie.implicits._ - import docspell.store.impl.Implicits._ - import docspell.store.impl.Column + import docspell.store.qb.DSL._ + import docspell.store.qb._ def latestUpdate(collective: Ident): ConnectionIO[Option[Timestamp]] = { - def max(col: Column, table: Fragment, cidCol: Column): Fragment = - selectSimple(col.max ++ fr"as t", table, cidCol.is(collective)) + def max_(col: Column[_], cidCol: Column[Ident]): Select = + Select(select(max(col).as("t")), from(col.table), cidCol === collective) - val equip = REquipment.as("e") - val sql = - List( - max( - ROrganization.Columns.updated, - ROrganization.table, - ROrganization.Columns.cid - ), - max(RPerson.Columns.updated, RPerson.table, RPerson.Columns.cid), - max( - equip.updated.oldColumn, - Fragment.const(equip.tableName), - equip.cid.oldColumn - ) - ) - .reduce(_ ++ fr"UNION ALL" ++ _) + val sql = union( + max_(ROrganization.T.updated, ROrganization.T.cid), + max_(RPerson.T.updated, RPerson.T.cid), + max_(REquipment.T.updated, REquipment.T.cid) + ) + val t = Column[Timestamp]("t", TableDef("")) - selectSimple(fr"MAX(t)", fr"(" ++ sql ++ fr") as x", Fragment.empty) + run(select(max(t)), fromSubSelect(sql).as("x")) .query[Option[Timestamp]] .option .map(_.flatten) diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 1e597efc..fbbaac6f 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -23,4 +23,9 @@ object DBFunction { def as(a: String) = copy(alias = a) } + + case class Max(column: Column[_], alias: String) extends DBFunction { + def as(a: String) = + copy(alias = a) + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 0b7b62a2..7eb12b55 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -9,9 +9,19 @@ import doobie.{Fragment, Put} trait DSL extends DoobieMeta { + def run(projection: Seq[SelectExpr], from: FromExpr): Fragment = + DoobieQuery(Select(projection, from, None)) + def run(projection: Seq[SelectExpr], from: FromExpr, where: Condition): Fragment = DoobieQuery(Select(projection, from, where)) + def runDistinct( + projection: Seq[SelectExpr], + from: FromExpr, + where: Condition + ): Fragment = + DoobieQuery.distinct(Select(projection, from, where)) + def select(dbf: DBFunction): Seq[SelectExpr] = Seq(SelectExpr.SelectFun(dbf)) @@ -21,12 +31,21 @@ trait DSL extends DoobieMeta { def select(seq: Seq[Column[_]], seqs: Seq[Column[_]]*): Seq[SelectExpr] = (seq ++ seqs.flatten).map(SelectExpr.SelectColumn.apply) - def from(table: TableDef): FromExpr = + def union(s1: Select, sn: Select*): Select = + Select.Union(s1, sn.toVector) + + def from(table: TableDef): FromExpr.From = FromExpr.From(table) + def fromSubSelect(sel: Select): FromExpr.SubSelect = + FromExpr.SubSelect(sel, "x") + def count(c: Column[_]): DBFunction = DBFunction.Count(c, "cn") + def max(c: Column[_]): DBFunction = + DBFunction.Max(c, "mn") + def and(c: Condition, cs: Condition*): Condition = c match { case Condition.And(head, tail) => diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index f066cccc..d78f3143 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -2,9 +2,9 @@ package docspell.store.qb sealed trait FromExpr { - def innerJoin(other: TableDef, on: Condition): FromExpr - - def leftJoin(other: TableDef, on: Condition): FromExpr +// def innerJoin(other: TableDef, on: Condition): FromExpr +// +// def leftJoin(other: TableDef, on: Condition): FromExpr } object FromExpr { @@ -23,6 +23,10 @@ object FromExpr { def leftJoin(other: TableDef, on: Condition): Joined = Joined(from, joins :+ Join.LeftJoin(other, on)) + } + case class SubSelect(sel: Select, name: String) extends FromExpr { + def as(name: String): SubSelect = + copy(name = name) } } diff --git a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala index 13da97cf..072d98c5 100644 --- a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala +++ b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala @@ -5,3 +5,12 @@ trait TableDef { def alias: Option[String] } + +object TableDef { + + def apply(table: String, aliasName: Option[String] = None): TableDef = + new TableDef { + def tableName: String = table + def alias: Option[String] = aliasName + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala index 39e10864..71ee3606 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala @@ -15,6 +15,9 @@ object FromExprBuilder { case FromExpr.Joined(from, joins) => build(from) ++ joins.map(buildJoin).foldLeft(Fragment.empty)(_ ++ _) + + case FromExpr.SubSelect(sel, name) => + sql" FROM (" ++ DoobieQuery(sel) ++ fr") AS" ++ Fragment.const(name) } def buildTable(table: TableDef): Fragment = diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala index e60308bd..f59d1fe5 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala @@ -13,16 +13,19 @@ object SelectExprBuilder { column(col) case SelectExpr.SelectFun(DBFunction.CountAll(alias)) => - fr"COUNT(*) AS" ++ Fragment.const(alias) + sql"COUNT(*) AS" ++ Fragment.const(alias) case SelectExpr.SelectFun(DBFunction.Count(col, alias)) => - fr"COUNT(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) + sql"COUNT(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) + + case SelectExpr.SelectFun(DBFunction.Max(col, alias)) => + sql"MAX(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) } def column(col: Column[_]): Fragment = { - val prefix = - Fragment.const0(col.table.alias.getOrElse(col.table.tableName)) - prefix ++ Fragment.const0(".") ++ Fragment.const0(col.name) + val prefix = col.table.alias.getOrElse(col.table.tableName) + if (prefix.isEmpty) columnNoPrefix(col) + else Fragment.const0(prefix) ++ Fragment.const0(".") ++ Fragment.const0(col.name) } def columnNoPrefix(col: Column[_]): Fragment = diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index a180623f..62b23ab0 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -91,61 +91,28 @@ object QCollective { sql.run.query[TagCount].to[List] } -// def tagCloud2(coll: Ident): ConnectionIO[List[TagCount]] = { -// val tagItem = RTagItem.as("r") -// val TC = RTag.Columns -// -// val q3 = fr"SELECT" ++ commas( -// TC.all.map(_.prefix("t").f) ++ Seq(fr"count(" ++ tagItem.itemId.column.f ++ fr")") -// ) ++ -// fr"FROM" ++ Fragment.const(tagItem.tableName) ++ fr"r" ++ -// fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ tagItem.tagId.column -// .prefix("r") -// .is(TC.tid.prefix("t")) ++ -// fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ -// fr"GROUP BY" ++ commas( -// TC.name.prefix("t").f, -// TC.tid.prefix("t").f, -// TC.category.prefix("t").f -// ) -// -// q3.query[TagCount].to[List] -// } def getContacts( coll: Ident, query: Option[String], kind: Option[ContactKind] ): Stream[ConnectionIO, RContact] = { - val RO = ROrganization - val RP = RPerson - val RC = RContact + import docspell.store.qb.DSL._ + import docspell.store.qb._ - val orgCond = selectSimple(Seq(RO.Columns.oid), RO.table, RO.Columns.cid.is(coll)) - val persCond = selectSimple(Seq(RP.Columns.pid), RP.table, RP.Columns.cid.is(coll)) - val queryCond = query match { - case Some(q) => - Seq(RC.Columns.value.lowerLike(s"%${q.toLowerCase}%")) - case None => - Seq.empty - } - val kindCond = kind match { - case Some(k) => - Seq(RC.Columns.kind.is(k)) - case None => - Seq.empty - } + val ro = ROrganization.as("o") + val rp = RPerson.as("p") + val rc = RContact.as("c") - val q = selectSimple( - RC.Columns.all, - RC.table, - and( - Seq( - or(RC.Columns.orgId.isIn(orgCond), RC.Columns.personId.isIn(persCond)) - ) ++ queryCond ++ kindCond - ) - ) ++ orderBy(RC.Columns.value.f) + val orgCond = Select(select(ro.oid), from(ro), ro.cid === coll) + val persCond = Select(select(rp.pid), from(rp), rp.cid === coll) + val valueFilter = query.map(s => rc.value.like(s"%${s.toLowerCase}%")) + val kindFilter = kind.map(k => rc.kind === k) - q.query[RContact].stream + Select( + select(rc.all), + from(rc), + (rc.orgId.in(orgCond) || rc.personId.in(persCond)) &&? valueFilter &&? kindFilter + ).orderBy(rc.value).run.query[RContact].stream } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 29d6ea89..b41ebfa6 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -88,13 +88,17 @@ object QItem { def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { val equip = REquipment.as("e") - val IC = RItem.Columns.all.map(_.prefix("i")) - val OC = ROrganization.Columns.all.map(_.prefix("o")) - val P0C = RPerson.Columns.all.map(_.prefix("p0")) - val P1C = RPerson.Columns.all.map(_.prefix("p1")) - val EC = equip.all.map(_.oldColumn).map(_.prefix("e")) - val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) - val FC = List(RFolder.Columns.id, RFolder.Columns.name).map(_.prefix("f")) + val org = ROrganization.as("o") + val pers0 = RPerson.as("p0") + val pers1 = RPerson.as("p1") + + val IC = RItem.Columns.all.map(_.prefix("i")) + val OC = org.all.map(_.column) + val P0C = pers0.all.map(_.column) + val P1C = pers1.all.map(_.column) + val EC = equip.all.map(_.oldColumn).map(_.prefix("e")) + val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) + val FC = List(RFolder.Columns.id, RFolder.Columns.name).map(_.prefix("f")) val cq = selectSimple( @@ -102,15 +106,21 @@ object QItem { RItem.table ++ fr"i", Fragment.empty ) ++ - fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg + fr"LEFT JOIN" ++ Fragment.const( + org.tableName + ) ++ fr"o ON" ++ RItem.Columns.corrOrg .prefix("i") - .is(ROrganization.Columns.oid.prefix("o")) ++ - fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson + .is(org.oid.column) ++ + fr"LEFT JOIN" ++ Fragment.const( + pers0.tableName + ) ++ fr"p0 ON" ++ RItem.Columns.corrPerson .prefix("i") - .is(RPerson.Columns.pid.prefix("p0")) ++ - fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson + .is(pers0.pid.column) ++ + fr"LEFT JOIN" ++ Fragment.const( + pers1.tableName + ) ++ fr"p1 ON" ++ RItem.Columns.concPerson .prefix("i") - .is(RPerson.Columns.pid.prefix("p1")) ++ + .is(pers1.pid.column) ++ fr"LEFT JOIN" ++ Fragment.const( equip.tableName ) ++ fr"e ON" ++ RItem.Columns.concEquipment @@ -308,15 +318,15 @@ object QItem { moreCols: Seq[Fragment], ctes: (String, Fragment)* ): Fragment = { - val equip = REquipment.as("e1") + val equip = REquipment.as("e1") + val org = ROrganization.as("o0") + val pers0 = RPerson.as("p0") + val pers1 = RPerson.as("p1") + val IC = RItem.Columns val AC = RAttachment.Columns - val PC = RPerson.Columns - val OC = ROrganization.Columns val FC = RFolder.Columns val itemCols = IC.all - val personCols = List(PC.pid, PC.name) - val orgCols = List(OC.oid, OC.name) val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) val folderCols = List(FC.id, FC.name) val cvItem = RCustomFieldValue.Columns.itemId.prefix("cv") @@ -332,12 +342,12 @@ object QItem { IC.incoming.prefix("i").f, IC.created.prefix("i").f, fr"COALESCE(a.num, 0)", - OC.oid.prefix("o0").f, - OC.name.prefix("o0").f, - PC.pid.prefix("p0").f, - PC.name.prefix("p0").f, - PC.pid.prefix("p1").f, - PC.name.prefix("p1").f, + org.oid.column.f, + org.name.column.f, + pers0.pid.column.f, + pers0.name.column.f, + pers1.pid.column.f, + pers1.name.column.f, equip.eid.oldColumn.prefix("e1").f, equip.name.oldColumn.prefix("e1").f, FC.id.prefix("f1").f, @@ -356,9 +366,17 @@ object QItem { val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.account.collective)) val withPerson = - selectSimple(personCols, RPerson.table, PC.cid.is(q.account.collective)) + selectSimple( + List(RPerson.T.pid.column, RPerson.T.name.column), + Fragment.const(RPerson.T.tableName), + RPerson.T.cid.column.is(q.account.collective) + ) val withOrgs = - selectSimple(orgCols, ROrganization.table, OC.cid.is(q.account.collective)) + selectSimple( + List(ROrganization.T.oid.column, ROrganization.T.name.column), + Fragment.const(ROrganization.T.tableName), + ROrganization.T.cid.column.is(q.account.collective) + ) val withEquips = selectSimple( equipCols, @@ -386,9 +404,9 @@ object QItem { ) ++ selectKW ++ finalCols ++ fr" FROM items i" ++ fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ - fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ - fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ - fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ + fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(pers0.pid.column) ++ + fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(org.oid.column) ++ + fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(pers1.pid.column) ++ fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment .prefix("i") .is(equip.eid.oldColumn.prefix("e1")) ++ @@ -404,9 +422,10 @@ object QItem { batch: Batch ): Stream[ConnectionIO, ListItem] = { val equip = REquipment.as("e1") + val org = ROrganization.as("o0") + val pers0 = RPerson.as("p0") + val pers1 = RPerson.as("p1") val IC = RItem.Columns - val PC = RPerson.Columns - val OC = ROrganization.Columns // inclusive tags are AND-ed val tagSelectsIncl = q.tagsInclude @@ -436,18 +455,18 @@ object QItem { allNames .map(n => or( - OC.name.prefix("o0").lowerLike(n), - PC.name.prefix("p0").lowerLike(n), - PC.name.prefix("p1").lowerLike(n), + org.name.column.lowerLike(n), + pers0.name.column.lowerLike(n), + pers1.name.column.lowerLike(n), equip.name.oldColumn.prefix("e1").lowerLike(n), IC.name.prefix("i").lowerLike(n), IC.notes.prefix("i").lowerLike(n) ) ) .getOrElse(Fragment.empty), - RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson), - ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg), - RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson), + pers0.pid.column.isOrDiscard(q.corrPerson), + org.oid.column.isOrDiscard(q.corrOrg), + pers1.pid.column.isOrDiscard(q.concPerson), equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip), RFolder.Columns.id.prefix("f1").isOrDiscard(q.folder), if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala index d0234973..6d41a841 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala @@ -4,10 +4,8 @@ import cats.implicits._ import fs2._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ -import docspell.store.records.ROrganization.{Columns => OC} -import docspell.store.records.RPerson.{Columns => PC} +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import docspell.store.{AddResult, Store} @@ -19,29 +17,22 @@ object QOrganization { def findOrgAndContact( coll: Ident, query: Option[String], - order: OC.type => Column + order: ROrganization.Table => Column[_] ): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = { - val oColl = ROrganization.Columns.cid.prefix("o") - val oName = ROrganization.Columns.name.prefix("o") - val oNotes = ROrganization.Columns.notes.prefix("o") - val oId = ROrganization.Columns.oid.prefix("o") - val cOrg = RContact.Columns.orgId.prefix("c") - val cVal = RContact.Columns.value.prefix("c") + val org = ROrganization.as("o") + val c = RContact.as("c") - val cols = ROrganization.Columns.all.map(_.prefix("o")) ++ RContact.Columns.all - .map(_.prefix("c")) - val from = ROrganization.table ++ fr"o LEFT JOIN" ++ - RContact.table ++ fr"c ON" ++ cOrg.is(oId) + val valFilter = query.map { q => + val v = s"%$q%" + c.value.like(v) || org.name.like(v) || org.notes.like(v) + } + val sql = Select( + select(org.all, c.all), + from(org).leftJoin(c, c.orgId === org.oid), + org.cid === coll &&? valFilter + ).orderBy(order(org)) - val q = Seq(oColl.is(coll)) ++ (query match { - case Some(str) => - val v = s"%$str%" - Seq(or(cVal.lowerLike(v), oName.lowerLike(v), oNotes.lowerLike(v))) - case None => - Seq.empty - }) - - (selectSimple(cols, from, and(q)) ++ orderBy(order(OC).prefix("o").f)) + sql.run .query[(ROrganization, Option[RContact])] .stream .groupAdjacentBy(_._1) @@ -55,18 +46,16 @@ object QOrganization { coll: Ident, orgId: Ident ): ConnectionIO[Option[(ROrganization, Vector[RContact])]] = { - val oColl = ROrganization.Columns.cid.prefix("o") - val oId = ROrganization.Columns.oid.prefix("o") - val cOrg = RContact.Columns.orgId.prefix("c") + val org = ROrganization.as("o") + val c = RContact.as("c") - val cols = ROrganization.Columns.all.map(_.prefix("o")) ++ RContact.Columns.all - .map(_.prefix("c")) - val from = ROrganization.table ++ fr"o LEFT JOIN" ++ - RContact.table ++ fr"c ON" ++ cOrg.is(oId) + val sql = run( + select(org.all, c.all), + from(org).leftJoin(c, c.orgId === org.oid), + org.cid === coll && org.oid === orgId + ) - val q = and(oColl.is(coll), oId.is(orgId)) - - selectSimple(cols, from, q) + sql .query[(ROrganization, Option[RContact])] .stream .groupAdjacentBy(_._1) @@ -81,33 +70,23 @@ object QOrganization { def findPersonAndContact( coll: Ident, query: Option[String], - order: PC.type => Column + order: RPerson.Table => Column[_] ): Stream[ConnectionIO, (RPerson, Option[ROrganization], Vector[RContact])] = { - val pColl = PC.cid.prefix("p") - val pName = RPerson.Columns.name.prefix("p") - val pNotes = RPerson.Columns.notes.prefix("p") - val pId = RPerson.Columns.pid.prefix("p") - val cPers = RContact.Columns.personId.prefix("c") - val cVal = RContact.Columns.value.prefix("c") - val oId = ROrganization.Columns.oid.prefix("o") - val pOid = RPerson.Columns.oid.prefix("p") + val pers = RPerson.as("p") + val org = ROrganization.as("o") + val c = RContact.as("c") + val valFilter = query + .map(s => s"%$s%") + .map(v => c.value.like(v) || pers.name.like(v) || pers.notes.like(v)) + val sql = Select( + select(pers.all, org.all, c.all), + from(pers) + .leftJoin(org, org.oid === pers.oid) + .leftJoin(c, c.personId === pers.pid), + pers.cid === coll &&? valFilter + ).orderBy(order(pers)) - val cols = RPerson.Columns.all.map(_.prefix("p")) ++ - ROrganization.Columns.all.map(_.prefix("o")) ++ - RContact.Columns.all.map(_.prefix("c")) - val from = RPerson.table ++ fr"p LEFT JOIN" ++ - ROrganization.table ++ fr"o ON" ++ pOid.is(oId) ++ fr"LEFT JOIN" ++ - RContact.table ++ fr"c ON" ++ cPers.is(pId) - - val q = Seq(pColl.is(coll)) ++ (query match { - case Some(str) => - val v = s"%${str.toLowerCase}%" - Seq(or(cVal.lowerLike(v), pName.lowerLike(v), pNotes.lowerLike(v))) - case None => - Seq.empty - }) - - (selectSimple(cols, from, and(q)) ++ orderBy(order(PC).prefix("p").f)) + sql.run .query[(RPerson, Option[ROrganization], Option[RContact])] .stream .groupAdjacentBy(_._1) @@ -122,22 +101,19 @@ object QOrganization { coll: Ident, persId: Ident ): ConnectionIO[Option[(RPerson, Option[ROrganization], Vector[RContact])]] = { - val pColl = PC.cid.prefix("p") - val pId = RPerson.Columns.pid.prefix("p") - val cPers = RContact.Columns.personId.prefix("c") - val oId = ROrganization.Columns.oid.prefix("o") - val pOid = RPerson.Columns.oid.prefix("p") + val pers = RPerson.as("p") + val org = ROrganization.as("o") + val c = RContact.as("c") + val sql = + run( + select(pers.all, org.all, c.all), + from(pers) + .leftJoin(org, pers.oid === org.oid) + .leftJoin(c, c.personId === pers.pid), + pers.cid === coll && pers.pid === persId + ) - val cols = RPerson.Columns.all.map(_.prefix("p")) ++ - ROrganization.Columns.all.map(_.prefix("o")) ++ - RContact.Columns.all.map(_.prefix("c")) - val from = RPerson.table ++ fr"p LEFT JOIN" ++ - ROrganization.table ++ fr"o ON" ++ pOid.is(oId) ++ fr"LEFT JOIN" ++ - RContact.table ++ fr"c ON" ++ cPers.is(pId) - - val q = and(pColl.is(coll), pId.is(persId)) - - selectSimple(cols, from, q) + sql .query[(RPerson, Option[ROrganization], Option[RContact])] .stream .groupAdjacentBy(_._1) @@ -156,23 +132,15 @@ object QOrganization { ck: Option[ContactKind], concerning: Option[Boolean] ): Stream[ConnectionIO, RPerson] = { - val pColl = PC.cid.prefix("p") - val pConc = PC.concerning.prefix("p") - val pId = PC.pid.prefix("p") - val cPers = RContact.Columns.personId.prefix("c") - val cVal = RContact.Columns.value.prefix("c") - val cKind = RContact.Columns.kind.prefix("c") - - val from = RPerson.table ++ fr"p INNER JOIN" ++ - RContact.table ++ fr"c ON" ++ cPers.is(pId) - val q = Seq( - cVal.lowerLike(s"%${value.toLowerCase}%"), - pColl.is(coll) - ) ++ concerning.map(pConc.is(_)).toSeq ++ ck.map(cKind.is(_)).toSeq - - selectDistinct(PC.all.map(_.prefix("p")), from, and(q)) - .query[RPerson] - .stream + val p = RPerson.as("p") + val c = RContact.as("c") + runDistinct( + select(p.all), + from(p).innerJoin(c, c.personId === p.pid), + c.value.like(s"%${value.toLowerCase}%") && p.cid === coll &&? + concerning.map(c => p.concerning === c) &&? + ck.map(k => c.kind === k) + ).query[RPerson].stream } def addOrg[F[_]]( diff --git a/modules/store/src/main/scala/docspell/store/records/RContact.scala b/modules/store/src/main/scala/docspell/store/records/RContact.scala index f6d3598a..7fd64778 100644 --- a/modules/store/src/main/scala/docspell/store/records/RContact.scala +++ b/modules/store/src/main/scala/docspell/store/records/RContact.scala @@ -1,8 +1,8 @@ package docspell.store.records import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -18,64 +18,62 @@ case class RContact( object RContact { - val table = fr"contact" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "contact" - object Columns { - val contactId = Column("contactid") - val value = Column("value") - val kind = Column("kind") - val personId = Column("pid") - val orgId = Column("oid") - val created = Column("created") + val contactId = Column[Ident]("contactid", this) + val value = Column[String]("value", this) + val kind = Column[ContactKind]("kind", this) + val personId = Column[Ident]("pid", this) + val orgId = Column[Ident]("oid", this) + val created = Column[Timestamp]("created", this) val all = List(contactId, value, kind, personId, orgId, created) } - import Columns._ + private val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RContact): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(v: RContact): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}" ) - sql.update.run - } - def update(v: RContact): ConnectionIO[Int] = { - val sql = updateRow( - table, - contactId.is(v.contactId), - commas( - value.setTo(v.value), - kind.setTo(v.kind), - personId.setTo(v.personId), - orgId.setTo(v.orgId) + def update(v: RContact): ConnectionIO[Int] = + DML.update( + T, + T.contactId === v.contactId, + DML.set( + T.value.setTo(v.value), + T.kind.setTo(v.kind), + T.personId.setTo(v.personId), + T.orgId.setTo(v.orgId) ) ) - sql.update.run - } def delete(v: RContact): ConnectionIO[Int] = - deleteFrom(table, contactId.is(v.contactId)).update.run + DML.delete(T, T.contactId === v.contactId) def deleteOrg(oid: Ident): ConnectionIO[Int] = - deleteFrom(table, orgId.is(oid)).update.run + DML.delete(T, T.orgId === oid) def deletePerson(pid: Ident): ConnectionIO[Int] = - deleteFrom(table, personId.is(pid)).update.run + DML.delete(T, T.personId === pid) def findById(id: Ident): ConnectionIO[Option[RContact]] = { - val sql = selectSimple(all, table, contactId.is(id)) + val sql = run(select(T.all), from(T), T.contactId === id) sql.query[RContact].option } def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = { - val sql = selectSimple(all, table, personId.is(pid)) + val sql = run(select(T.all), from(T), T.personId === pid) sql.query[RContact].to[Vector] } def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = { - val sql = selectSimple(all, table, orgId.is(oid)) + val sql = run(select(T.all), from(T), T.orgId === oid) sql.query[RContact].to[Vector] } } diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala index 7883731e..3e0276ea 100644 --- a/modules/store/src/main/scala/docspell/store/records/REquipment.scala +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -27,6 +27,7 @@ object REquipment { val all = List(eid, cid, name, created, updated) } + val T = Table(None) def as(alias: String): Table = Table(Some(alias)) diff --git a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala index 76a55d2a..2aa0a743 100644 --- a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala +++ b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala @@ -4,8 +4,8 @@ import cats.Eq import fs2.Stream import docspell.common.{IdRef, _} -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -27,73 +27,73 @@ object ROrganization { implicit val orgEq: Eq[ROrganization] = Eq.by[ROrganization, Ident](_.oid) - val table = fr"organization" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "organization" - object Columns { - val oid = Column("oid") - val cid = Column("cid") - val name = Column("name") - val street = Column("street") - val zip = Column("zip") - val city = Column("city") - val country = Column("country") - val notes = Column("notes") - val created = Column("created") - val updated = Column("updated") + val oid = Column[Ident]("oid", this) + val cid = Column[Ident]("cid", this) + val name = Column[String]("name", this) + val street = Column[String]("street", this) + val zip = Column[String]("zip", this) + val city = Column[String]("city", this) + val country = Column[String]("country", this) + val notes = Column[String]("notes", this) + val created = Column[Timestamp]("created", this) + val updated = Column[Timestamp]("updated", this) val all = List(oid, cid, name, street, zip, city, country, notes, created, updated) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: ROrganization): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(v: ROrganization): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created},${v.updated}" ) - sql.update.run - } def update(v: ROrganization): ConnectionIO[Int] = { def sql(now: Timestamp) = - updateRow( - table, - and(oid.is(v.oid), cid.is(v.cid)), - commas( - cid.setTo(v.cid), - name.setTo(v.name), - street.setTo(v.street), - zip.setTo(v.zip), - city.setTo(v.city), - country.setTo(v.country), - notes.setTo(v.notes), - updated.setTo(now) + DML.update( + T, + T.oid === v.oid && T.cid === v.cid, + DML.set( + T.cid.setTo(v.cid), + T.name.setTo(v.name), + T.street.setTo(v.street), + T.zip.setTo(v.zip), + T.city.setTo(v.city), + T.country.setTo(v.country), + T.notes.setTo(v.notes), + T.updated.setTo(now) ) ) for { now <- Timestamp.current[ConnectionIO] - n <- sql(now).update.run + n <- sql(now) } yield n } def existsByName(coll: Ident, oname: String): ConnectionIO[Boolean] = - selectCount(oid, table, and(cid.is(coll), name.is(oname))) + run(select(count(T.oid)), from(T), T.cid === coll && T.name === oname) .query[Int] .unique .map(_ > 0) def findById(id: Ident): ConnectionIO[Option[ROrganization]] = { - val sql = selectSimple(all, table, cid.is(id)) + val sql = run(select(T.all), from(T), T.cid === id) sql.query[ROrganization].option } def find(coll: Ident, orgName: String): ConnectionIO[Option[ROrganization]] = { - val sql = selectSimple(all, table, and(cid.is(coll), name.is(orgName))) + val sql = run(select(T.all), from(T), T.cid === coll && T.name === orgName) sql.query[ROrganization].option } def findLike(coll: Ident, orgName: String): ConnectionIO[Vector[IdRef]] = - selectSimple(List(oid, name), table, and(cid.is(coll), name.lowerLike(orgName))) + run(select(T.oid, T.name), from(T), T.cid === coll && T.name.like(orgName)) .query[IdRef] .to[Vector] @@ -102,42 +102,38 @@ object ROrganization { contactKind: ContactKind, value: String ): ConnectionIO[Vector[IdRef]] = { - val CC = RContact.Columns - val q = fr"SELECT DISTINCT" ++ commas(oid.prefix("o").f, name.prefix("o").f) ++ - fr"FROM" ++ table ++ fr"o" ++ - fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.orgId - .prefix("c") - .is(oid.prefix("o")) ++ - fr"WHERE" ++ and( - cid.prefix("o").is(coll), - CC.kind.prefix("c").is(contactKind), - CC.value.prefix("c").lowerLike(value) + val c = RContact.as("c") + val o = ROrganization.as("o") + runDistinct( + select(o.oid, o.name), + from(o).innerJoin(c, c.orgId === o.oid), + where( + o.cid === coll, + c.kind === contactKind, + c.value.like(value) ) - - q.query[IdRef].to[Vector] + ).query[IdRef].to[Vector] } def findAll( coll: Ident, - order: Columns.type => Column + order: Table => Column[_] ): Stream[ConnectionIO, ROrganization] = { - val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) - sql.query[ROrganization].stream + val sql = Select(select(T.all), from(T), T.cid === coll).orderBy(order(T)) + sql.run.query[ROrganization].stream } def findAllRef( coll: Ident, nameQ: Option[String], - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[IdRef]] = { - val q = Seq(cid.is(coll)) ++ (nameQ match { - case Some(str) => Seq(name.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) - val sql = selectSimple(List(oid, name), table, and(q)) ++ orderBy(order(Columns).f) - sql.query[IdRef].to[Vector] + val nameFilter = nameQ.map(s => T.name.like(s"%${s.toLowerCase}%")) + val sql = Select(select(T.oid, T.name), from(T), T.cid === coll &&? nameFilter) + .orderBy(order(T)) + sql.run.query[IdRef].to[Vector] } def delete(id: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(oid.is(id), cid.is(coll))).update.run + DML.delete(T, T.oid === id && T.cid === coll) } diff --git a/modules/store/src/main/scala/docspell/store/records/RPerson.scala b/modules/store/src/main/scala/docspell/store/records/RPerson.scala index c7df6fed..04eb7831 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPerson.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPerson.scala @@ -6,8 +6,8 @@ import cats.effect._ import fs2.Stream import docspell.common.{IdRef, _} -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -31,21 +31,21 @@ object RPerson { implicit val personEq: Eq[RPerson] = Eq.by(_.pid) - val table = fr"person" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "person" - object Columns { - val pid = Column("pid") - val cid = Column("cid") - val name = Column("name") - val street = Column("street") - val zip = Column("zip") - val city = Column("city") - val country = Column("country") - val notes = Column("notes") - val concerning = Column("concerning") - val created = Column("created") - val updated = Column("updated") - val oid = Column("oid") + val pid = Column[Ident]("pid", this) + val cid = Column[Ident]("cid", this) + val name = Column[String]("name", this) + val street = Column[String]("street", this) + val zip = Column[String]("zip", this) + val city = Column[String]("city", this) + val country = Column[String]("country", this) + val notes = Column[String]("notes", this) + val concerning = Column[Boolean]("concerning", this) + val created = Column[Timestamp]("created", this) + val updated = Column[Timestamp]("updated", this) + val oid = Column[Ident]("oid", this) val all = List( pid, cid, @@ -62,54 +62,54 @@ object RPerson { ) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RPerson): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(v: RPerson): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created},${v.updated},${v.oid}" ) - sql.update.run - } def update(v: RPerson): ConnectionIO[Int] = { def sql(now: Timestamp) = - updateRow( - table, - and(pid.is(v.pid), cid.is(v.cid)), - commas( - cid.setTo(v.cid), - name.setTo(v.name), - street.setTo(v.street), - zip.setTo(v.zip), - city.setTo(v.city), - country.setTo(v.country), - concerning.setTo(v.concerning), - notes.setTo(v.notes), - oid.setTo(v.oid), - updated.setTo(now) + DML.update( + T, + T.pid === v.pid && T.cid === v.cid, + DML.set( + T.cid.setTo(v.cid), + T.name.setTo(v.name), + T.street.setTo(v.street), + T.zip.setTo(v.zip), + T.city.setTo(v.city), + T.country.setTo(v.country), + T.concerning.setTo(v.concerning), + T.notes.setTo(v.notes), + T.oid.setTo(v.oid), + T.updated.setTo(now) ) ) for { now <- Timestamp.current[ConnectionIO] - n <- sql(now).update.run + n <- sql(now) } yield n } def existsByName(coll: Ident, pname: String): ConnectionIO[Boolean] = - selectCount(pid, table, and(cid.is(coll), name.is(pname))) + run(select(count(T.pid)), from(T), T.cid === coll && T.name === pname) .query[Int] .unique .map(_ > 0) def findById(id: Ident): ConnectionIO[Option[RPerson]] = { - val sql = selectSimple(all, table, cid.is(id)) + val sql = run(select(T.all), from(T), T.cid === id) sql.query[RPerson].option } def find(coll: Ident, personName: String): ConnectionIO[Option[RPerson]] = { - val sql = selectSimple(all, table, and(cid.is(coll), name.is(personName))) + val sql = run(select(T.all), from(T), T.cid === coll && T.name === personName) sql.query[RPerson].option } @@ -118,10 +118,10 @@ object RPerson { personName: String, concerningOnly: Boolean ): ConnectionIO[Vector[IdRef]] = - selectSimple( - List(pid, name), - table, - and(cid.is(coll), concerning.is(concerningOnly), name.lowerLike(personName)) + run( + select(T.pid, T.name), + from(T), + where(T.cid === coll, T.concerning === concerningOnly, T.name.like(personName)) ).query[IdRef].to[Vector] def findLike( @@ -130,53 +130,52 @@ object RPerson { value: String, concerningOnly: Boolean ): ConnectionIO[Vector[IdRef]] = { - val CC = RContact.Columns - val q = fr"SELECT DISTINCT" ++ commas(pid.prefix("p").f, name.prefix("p").f) ++ - fr"FROM" ++ table ++ fr"p" ++ - fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.personId - .prefix("c") - .is(pid.prefix("p")) ++ - fr"WHERE" ++ and( - cid.prefix("p").is(coll), - CC.kind.prefix("c").is(contactKind), - concerning.prefix("p").is(concerningOnly), - CC.value.prefix("c").lowerLike(value) - ) + val p = RPerson.as("p") + val c = RContact.as("c") - q.query[IdRef].to[Vector] + runDistinct( + select(p.pid, p.name), + from(p).innerJoin(c, p.pid === c.personId), + where( + p.cid === coll, + c.kind === contactKind, + p.concerning === concerningOnly, + c.value.like(value) + ) + ).query[IdRef].to[Vector] } def findAll( coll: Ident, - order: Columns.type => Column + order: Table => Column[_] ): Stream[ConnectionIO, RPerson] = { - val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) - sql.query[RPerson].stream + val sql = Select(select(T.all), from(T), T.cid === coll).orderBy(order(T)) + sql.run.query[RPerson].stream } def findAllRef( coll: Ident, nameQ: Option[String], - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[IdRef]] = { - val q = Seq(cid.is(coll)) ++ (nameQ match { - case Some(str) => Seq(name.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) - val sql = selectSimple(List(pid, name), table, and(q)) ++ orderBy(order(Columns).f) - sql.query[IdRef].to[Vector] + + val nameFilter = nameQ.map(s => T.name.like(s"%${s.toLowerCase}%")) + + val sql = Select(select(T.pid, T.name), from(T), T.cid === coll &&? nameFilter) + .orderBy(order(T)) + sql.run.query[IdRef].to[Vector] } def delete(personId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(pid.is(personId), cid.is(coll))).update.run + DML.delete(T, T.pid === personId && T.cid === coll) - def findOrganization(ids: Set[Ident]): ConnectionIO[Vector[PersonRef]] = { - val cols = Seq(pid, name, oid) + def findOrganization(ids: Set[Ident]): ConnectionIO[Vector[PersonRef]] = NonEmptyList.fromList(ids.toList) match { case Some(nel) => - selectSimple(cols, table, pid.isIn(nel)).query[PersonRef].to[Vector] + run(select(T.pid, T.name, T.oid), from(T), T.pid.in(nel)) + .query[PersonRef] + .to[Vector] case None => Sync[ConnectionIO].pure(Vector.empty) } - } } diff --git a/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala b/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala index 36d0b4f9..464c0576 100644 --- a/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala +++ b/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala @@ -4,8 +4,8 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -13,18 +13,20 @@ import doobie.implicits._ case class RRememberMe(id: Ident, accountId: AccountId, created: Timestamp, uses: Int) {} object RRememberMe { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "rememberme" - val table = fr"rememberme" - - object Columns { - val id = Column("id") - val cid = Column("cid") - val username = Column("login") - val created = Column("created") - val uses = Column("uses") + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val username = Column[Ident]("login", this) + val created = Column[Timestamp]("created", this) + val uses = Column[Int]("uses", this) val all = List(id, cid, username, created, uses) } - import Columns._ + + private val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def generate[F[_]: Sync](account: AccountId): F[RRememberMe] = for { @@ -33,29 +35,29 @@ object RRememberMe { } yield RRememberMe(i, account, c, 0) def insert(v: RRememberMe): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.accountId.collective},${v.accountId.user},${v.created},${v.uses}" - ).update.run + ) def insertNew(acc: AccountId): ConnectionIO[RRememberMe] = generate[ConnectionIO](acc).flatMap(v => insert(v).map(_ => v)) def findById(rid: Ident): ConnectionIO[Option[RRememberMe]] = - selectSimple(all, table, id.is(rid)).query[RRememberMe].option + run(select(T.all), from(T), T.id === rid).query[RRememberMe].option def delete(rid: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(rid)).update.run + DML.delete(T, T.id === rid) def incrementUse(rid: Ident): ConnectionIO[Int] = - updateRow(table, id.is(rid), uses.increment(1)).update.run + DML.update(T, T.id === rid, DML.set(T.uses.increment(1))) def useRememberMe( rid: Ident, minCreated: Timestamp ): ConnectionIO[Option[RRememberMe]] = { - val get = selectSimple(all, table, and(id.is(rid), created.isGt(minCreated))) + val get = run(select(T.all), from(T), T.id === rid && T.created > minCreated) .query[RRememberMe] .option for { @@ -65,5 +67,5 @@ object RRememberMe { } def deleteOlderThan(ts: Timestamp): ConnectionIO[Int] = - deleteFrom(table, created.isLt(ts)).update.run + DML.delete(T, T.created < ts) } From 1aa1f4367e99f59cee52b1e0fed4d7a384600c37 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 11 Dec 2020 23:15:18 +0100 Subject: [PATCH 09/38] Convert periodic tasks --- .../scala/docspell/store/qb/Condition.scala | 2 + .../main/scala/docspell/store/qb/DSL.scala | 8 +- .../scala/docspell/store/qb/Operator.scala | 1 + .../store/qb/impl/ConditionBuilder.scala | 8 ++ .../store/queries/QPeriodicTask.scala | 65 +++++++-------- .../docspell/store/queries/QUserTask.scala | 70 ++++++++-------- .../store/records/RPeriodicTask.scala | 82 +++++++++---------- 7 files changed, 126 insertions(+), 110 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 88a5487f..c1f623cb 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -20,6 +20,8 @@ object Condition { val P: Put[A] ) extends Condition + case class IsNull(col: Column[_]) extends Condition + case class And(c: Condition, cs: Vector[Condition]) extends Condition case class Or(c: Condition, cs: Vector[Condition]) extends Condition case class Not(c: Condition) extends Condition diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 7eb12b55..e399515b 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -94,14 +94,12 @@ trait DSL extends DoobieMeta { def ===(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Eq, value) - //TODO find some better way around the cast def ====(value: String): Condition = Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.Eq, value) def like(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.LowerLike, value) - //TODO find some better way around the cast def likes(value: String): Condition = Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.LowerLike, value) @@ -117,6 +115,9 @@ trait DSL extends DoobieMeta { def <(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Lt, value) + def <>(value: A)(implicit P: Put[A]): Condition = + Condition.CompareVal(col, Operator.Neq, value) + def in(subsel: Select): Condition = Condition.InSubSelect(col, subsel) @@ -126,6 +127,9 @@ trait DSL extends DoobieMeta { def inLower(values: NonEmptyList[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, true) + def isNull: Condition = + Condition.IsNull(col) + def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) } diff --git a/modules/store/src/main/scala/docspell/store/qb/Operator.scala b/modules/store/src/main/scala/docspell/store/qb/Operator.scala index c05559ca..907c2593 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Operator.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Operator.scala @@ -5,6 +5,7 @@ sealed trait Operator object Operator { case object Eq extends Operator + case object Neq extends Operator case object Gt extends Operator case object Lt extends Operator case object Gte extends Operator diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index 1f99df1e..2dfbd8a8 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -44,6 +44,9 @@ object ConditionBuilder { .map(a => buildValue(a)(c.P)) .reduce(_ ++ comma ++ _) ++ sql")" + case Condition.IsNull(col) => + SelectExprBuilder.column(col) ++ fr" is null" + case Condition.And(c, cs) => val inner = cs.prepended(c).map(build).reduce(_ ++ and ++ _) if (cs.isEmpty) inner @@ -54,6 +57,9 @@ object ConditionBuilder { if (cs.isEmpty) inner else parenOpen ++ inner ++ parenClose + case Condition.Not(Condition.IsNull(col)) => + SelectExprBuilder.column(col) ++ fr" is not null" + case Condition.Not(c) => fr"NOT" ++ build(c) } @@ -62,6 +68,8 @@ object ConditionBuilder { op match { case Operator.Eq => fr" =" + case Operator.Neq => + fr" <>" case Operator.Gt => fr" >" case Operator.Lt => diff --git a/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala index cf8451c5..46d8a273 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala @@ -1,7 +1,8 @@ package docspell.store.queries import docspell.common._ -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ @@ -9,47 +10,47 @@ import doobie.implicits._ object QPeriodicTask { - def clearWorkers(name: Ident): ConnectionIO[Int] = { - val worker = RPeriodicTask.Columns.worker - updateRow(RPeriodicTask.table, worker.is(name), worker.setTo[Ident](None)).update.run - } + private val RT = RPeriodicTask.T - def setWorker(pid: Ident, name: Ident, ts: Timestamp): ConnectionIO[Int] = { - val id = RPeriodicTask.Columns.id - val worker = RPeriodicTask.Columns.worker - val marked = RPeriodicTask.Columns.marked - updateRow( - RPeriodicTask.table, - and(id.is(pid), worker.isNull), - commas(worker.setTo(name), marked.setTo(ts)) - ).update.run - } + def clearWorkers(name: Ident): ConnectionIO[Int] = + DML.update( + RT, + RT.worker === name, + DML.set(RT.worker.setTo(None: Option[Ident])) + ) + + def setWorker(pid: Ident, name: Ident, ts: Timestamp): ConnectionIO[Int] = + DML + .update( + RT, + RT.id === pid && RT.worker.isNull, + DML.set( + RT.worker.setTo(name), + RT.marked.setTo(ts) + ) + ) def unsetWorker( pid: Ident, nextRun: Option[Timestamp] - ): ConnectionIO[Int] = { - val id = RPeriodicTask.Columns.id - val worker = RPeriodicTask.Columns.worker - val next = RPeriodicTask.Columns.nextrun - updateRow( - RPeriodicTask.table, - id.is(pid), - commas(worker.setTo[Ident](None), next.setTo(nextRun)) - ).update.run - } + ): ConnectionIO[Int] = + DML.update( + RT, + RT.id === pid, + DML.set( + RT.worker.setTo(None), + RT.nextrun.setTo(nextRun) + ) + ) def findNext(excl: Option[Ident]): ConnectionIO[Option[RPeriodicTask]] = { - val enabled = RPeriodicTask.Columns.enabled - val pid = RPeriodicTask.Columns.id - val order = orderBy(RPeriodicTask.Columns.nextrun.f) ++ fr"ASC" - val where = excl match { - case Some(id) => and(pid.isNot(id), enabled.is(true)) - case None => enabled.is(true) + case Some(id) => RT.id <> id && RT.enabled === true + case None => RT.enabled === true } val sql = - selectSimple(RPeriodicTask.Columns.all, RPeriodicTask.table, where) ++ order + Select(select(RT.all), from(RT), where).orderBy(RT.nextrun.asc).run + sql.query[RPeriodicTask].streamWithChunkSize(2).take(1).compile.last } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala b/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala index 9ef601fa..13fbbc89 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala @@ -3,33 +3,34 @@ package docspell.store.queries import fs2._ import docspell.common._ -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import docspell.store.usertask.UserTask import doobie._ object QUserTask { - private val cols = RPeriodicTask.Columns + private val RT = RPeriodicTask.T def findAll(account: AccountId): Stream[ConnectionIO, UserTask[String]] = - selectSimple( - RPeriodicTask.Columns.all, - RPeriodicTask.table, - and(cols.group.is(account.collective), cols.submitter.is(account.user)) + run( + select(RT.all), + from(RT), + RT.group === account.collective && RT.submitter === account.user ).query[RPeriodicTask].stream.map(makeUserTask) def findByName( account: AccountId, name: Ident ): Stream[ConnectionIO, UserTask[String]] = - selectSimple( - RPeriodicTask.Columns.all, - RPeriodicTask.table, - and( - cols.group.is(account.collective), - cols.submitter.is(account.user), - cols.task.is(name) + run( + select(RT.all), + from(RT), + where( + RT.group === account.collective, + RT.submitter === account.user, + RT.task === name ) ).query[RPeriodicTask].stream.map(makeUserTask) @@ -37,13 +38,13 @@ object QUserTask { account: AccountId, id: Ident ): ConnectionIO[Option[UserTask[String]]] = - selectSimple( - RPeriodicTask.Columns.all, - RPeriodicTask.table, - and( - cols.group.is(account.collective), - cols.submitter.is(account.user), - cols.id.is(id) + run( + select(RT.all), + from(RT), + where( + RT.group === account.collective, + RT.submitter === account.user, + RT.id === id ) ).query[RPeriodicTask].option.map(_.map(makeUserTask)) @@ -63,24 +64,25 @@ object QUserTask { RPeriodicTask.exists(id) def delete(account: AccountId, id: Ident): ConnectionIO[Int] = - deleteFrom( - RPeriodicTask.table, - and( - cols.group.is(account.collective), - cols.submitter.is(account.user), - cols.id.is(id) + DML + .delete( + RT, + where( + RT.group === account.collective, + RT.submitter === account.user, + RT.id === id + ) ) - ).update.run def deleteAll(account: AccountId, name: Ident): ConnectionIO[Int] = - deleteFrom( - RPeriodicTask.table, - and( - cols.group.is(account.collective), - cols.submitter.is(account.user), - cols.task.is(name) + DML.delete( + RT, + where( + RT.group === account.collective, + RT.submitter === account.user, + RT.task === name ) - ).update.run + ) def makeUserTask(r: RPeriodicTask): UserTask[String] = UserTask(r.id, r.task, r.enabled, r.timer, r.args) diff --git a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala index 4f7c68a3..e0dcdb3f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala @@ -4,8 +4,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import com.github.eikek.calev.CalEvent import doobie._ @@ -107,22 +107,22 @@ object RPeriodicTask { )(implicit E: Encoder[A]): F[RPeriodicTask] = create[F](enabled, task, group, E(args).noSpaces, subject, submitter, priority, timer) - val table = fr"periodic_task" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "periodic_task" - object Columns { - val id = Column("id") - val enabled = Column("enabled") - val task = Column("task") - val group = Column("group_") - val args = Column("args") - val subject = Column("subject") - val submitter = Column("submitter") - val priority = Column("priority") - val worker = Column("worker") - val marked = Column("marked") - val timer = Column("timer") - val nextrun = Column("nextrun") - val created = Column("created") + val id = Column[Ident]("id", this) + val enabled = Column[Boolean]("enabled", this) + val task = Column[Ident]("task", this) + val group = Column[Ident]("group_", this) + val args = Column[String]("args", this) + val subject = Column[String]("subject", this) + val submitter = Column[Ident]("submitter", this) + val priority = Column[Priority]("priority", this) + val worker = Column[Ident]("worker", this) + val marked = Column[Timestamp]("marked", this) + val timer = Column[CalEvent]("timer", this) + val nextrun = Column[Timestamp]("nextrun", this) + val created = Column[Timestamp]("created", this) val all = List( id, enabled, @@ -140,39 +140,37 @@ object RPeriodicTask { ) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(v: RPeriodicTask): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(v: RPeriodicTask): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${v.id},${v.enabled},${v.task},${v.group},${v.args}," ++ fr"${v.subject},${v.submitter},${v.priority},${v.worker}," ++ fr"${v.marked},${v.timer},${v.nextrun},${v.created}" ) - sql.update.run - } - def update(v: RPeriodicTask): ConnectionIO[Int] = { - val sql = updateRow( - table, - id.is(v.id), - commas( - enabled.setTo(v.enabled), - group.setTo(v.group), - args.setTo(v.args), - subject.setTo(v.subject), - submitter.setTo(v.submitter), - priority.setTo(v.priority), - worker.setTo(v.worker), - marked.setTo(v.marked), - timer.setTo(v.timer), - nextrun.setTo(v.nextrun) + def update(v: RPeriodicTask): ConnectionIO[Int] = + DML.update( + T, + T.id === v.id, + DML.set( + T.enabled.setTo(v.enabled), + T.group.setTo(v.group), + T.args.setTo(v.args), + T.subject.setTo(v.subject), + T.submitter.setTo(v.submitter), + T.priority.setTo(v.priority), + T.worker.setTo(v.worker), + T.marked.setTo(v.marked), + T.timer.setTo(v.timer), + T.nextrun.setTo(v.nextrun) ) ) - sql.update.run - } def exists(pid: Ident): ConnectionIO[Boolean] = - selectCount(id, table, id.is(pid)).query[Int].unique.map(_ > 0) + run(select(count(T.id)), from(T), T.id === pid).query[Int].unique.map(_ > 0) } From e3f6892abd7b505524f03946ce8a3cdd1448216b Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sat, 12 Dec 2020 01:56:06 +0100 Subject: [PATCH 10/38] Convert job record --- .../scala/docspell/backend/ops/OJob.scala | 2 +- .../main/scala/docspell/common/JobState.scala | 22 +- .../docspell/joex/analysis/RegexNerFile.scala | 2 +- .../scala/docspell/store/impl/Implicits.scala | 11 +- .../main/scala/docspell/store/qb/Column.scala | 5 +- .../scala/docspell/store/qb/Condition.scala | 4 + .../scala/docspell/store/qb/DBFunction.scala | 38 ++- .../main/scala/docspell/store/qb/DSL.scala | 64 ++++- .../scala/docspell/store/qb/GroupBy.scala | 4 +- .../main/scala/docspell/store/qb/Select.scala | 15 +- .../scala/docspell/store/qb/SelectExpr.scala | 22 +- .../store/qb/impl/CommonBuilder.scala | 20 ++ .../store/qb/impl/ConditionBuilder.scala | 20 +- .../store/qb/impl/DBFunctionBuilder.scala | 40 +++ .../docspell/store/qb/impl/DoobieQuery.scala | 2 + .../store/qb/impl/SelectExprBuilder.scala | 25 +- .../scala/docspell/store/queries/QJob.scala | 133 +++++----- .../scala/docspell/store/records/RJob.scala | 249 +++++++++--------- .../docspell/store/records/RJobGroupUse.scala | 24 +- .../docspell/store/records/RJobLog.scala | 40 +-- 20 files changed, 445 insertions(+), 297 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala create mode 100644 modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala index 4b83a30a..6809f259 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala @@ -39,7 +39,7 @@ object OJob { def queued: Vector[JobDetail] = jobs.filter(r => JobState.queued.contains(r.job.state)) def done: Vector[JobDetail] = - jobs.filter(r => JobState.done.contains(r.job.state)) + jobs.filter(r => JobState.done.toList.contains(r.job.state)) def running: Vector[JobDetail] = jobs.filter(_.job.state == JobState.Running) } diff --git a/modules/common/src/main/scala/docspell/common/JobState.scala b/modules/common/src/main/scala/docspell/common/JobState.scala index 69f0e622..0ab68584 100644 --- a/modules/common/src/main/scala/docspell/common/JobState.scala +++ b/modules/common/src/main/scala/docspell/common/JobState.scala @@ -1,5 +1,7 @@ package docspell.common +import cats.data.NonEmptyList + import io.circe.{Decoder, Encoder} sealed trait JobState { self: Product => @@ -12,8 +14,6 @@ object JobState { /** Waiting for being executed. */ case object Waiting extends JobState {} - def waiting: JobState = Waiting - /** A scheduler has picked up this job and will pass it to the next * free slot. */ @@ -34,10 +34,20 @@ object JobState { /** Finished with success */ case object Success extends JobState {} - val all: Set[JobState] = - Set(Waiting, Scheduled, Running, Stuck, Failed, Cancelled, Success) - val queued: Set[JobState] = Set(Waiting, Scheduled, Stuck) - val done: Set[JobState] = Set(Failed, Cancelled, Success) + val waiting: JobState = Waiting + val stuck: JobState = Stuck + val scheduled: JobState = Scheduled + val running: JobState = Running + val failed: JobState = Failed + val cancelled: JobState = Cancelled + val success: JobState = Success + + val all: NonEmptyList[JobState] = + NonEmptyList.of(Waiting, Scheduled, Running, Stuck, Failed, Cancelled, Success) + val queued: Set[JobState] = Set(Waiting, Scheduled, Stuck) + val done: NonEmptyList[JobState] = NonEmptyList.of(Failed, Cancelled, Success) + val notDone: NonEmptyList[JobState] = //all - done + NonEmptyList.of(Waiting, Scheduled, Running, Stuck) val inProgress: Set[JobState] = Set(Scheduled, Running, Stuck) def parse(str: String): Either[String, JobState] = diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index fb5f097c..e5dcce3e 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -141,7 +141,7 @@ object RegexNerFile { def latestUpdate(collective: Ident): ConnectionIO[Option[Timestamp]] = { def max_(col: Column[_], cidCol: Column[Ident]): Select = - Select(select(max(col).as("t")), from(col.table), cidCol === collective) + Select(List(max(col).as("t")), from(col.table), cidCol === collective) val sql = union( max_(ROrganization.T.updated, ROrganization.T.cid), diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala index 30cba7ca..2047b301 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala @@ -6,15 +6,10 @@ object Implicits extends DoobieMeta with DoobieSyntax { def oldColumn: Column = Column(col.name) - def column: Column = { - val c = col.alias match { - case Some(a) => oldColumn.as(a) + def column: Column = + col.table.alias match { + case Some(p) => oldColumn.prefix(p) case None => oldColumn } - col.table.alias match { - case Some(p) => c.prefix(p) - case None => c - } - } } } diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala index 90936f90..a8465417 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Column.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -1,8 +1,5 @@ package docspell.store.qb -case class Column[A](name: String, table: TableDef, alias: Option[String] = None) { - def as(alias: String): Column[A] = - copy(alias = Some(alias)) -} +case class Column[A](name: String, table: TableDef) object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index c1f623cb..4738545a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -12,6 +12,10 @@ object Condition { val P: Put[A] ) extends Condition + case class CompareFVal[A](dbf: DBFunction, op: Operator, value: A)(implicit + val P: Put[A] + ) extends Condition + case class CompareCol[A](col1: Column[A], op: Operator, col2: Column[A]) extends Condition diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index fbbaac6f..52b50024 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -1,31 +1,27 @@ package docspell.store.qb -sealed trait DBFunction { - def alias: String - - def as(alias: String): DBFunction -} +sealed trait DBFunction {} object DBFunction { - def countAllAs(alias: String) = - CountAll(alias) + val countAll: DBFunction = CountAll - def countAs[A](column: Column[A], alias: String): DBFunction = - Count(column, alias) + def countAs[A](column: Column[A]): DBFunction = + Count(column) - case class CountAll(alias: String) extends DBFunction { - def as(a: String) = - copy(alias = a) - } + case object CountAll extends DBFunction - case class Count(column: Column[_], alias: String) extends DBFunction { - def as(a: String) = - copy(alias = a) - } + case class Count(column: Column[_]) extends DBFunction - case class Max(column: Column[_], alias: String) extends DBFunction { - def as(a: String) = - copy(alias = a) - } + case class Max(column: Column[_]) extends DBFunction + + case class Min(column: Column[_]) extends DBFunction + + case class Coalesce(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction + + case class Power(expr: SelectExpr, base: Int) extends DBFunction + + case class Plus(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction + + case class Mult(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index e399515b..849305c4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -23,13 +23,13 @@ trait DSL extends DoobieMeta { DoobieQuery.distinct(Select(projection, from, where)) def select(dbf: DBFunction): Seq[SelectExpr] = - Seq(SelectExpr.SelectFun(dbf)) + Seq(SelectExpr.SelectFun(dbf, None)) def select(c: Column[_], cs: Column[_]*): Seq[SelectExpr] = select(c :: cs.toList) def select(seq: Seq[Column[_]], seqs: Seq[Column[_]]*): Seq[SelectExpr] = - (seq ++ seqs.flatten).map(SelectExpr.SelectColumn.apply) + (seq ++ seqs.flatten).map(c => SelectExpr.SelectColumn(c, None)) def union(s1: Select, sn: Select*): Select = Select.Union(s1, sn.toVector) @@ -41,10 +41,28 @@ trait DSL extends DoobieMeta { FromExpr.SubSelect(sel, "x") def count(c: Column[_]): DBFunction = - DBFunction.Count(c, "cn") + DBFunction.Count(c) def max(c: Column[_]): DBFunction = - DBFunction.Max(c, "mn") + DBFunction.Max(c) + + def min(c: Column[_]): DBFunction = + DBFunction.Min(c) + + def coalesce(expr: SelectExpr, more: SelectExpr*): DBFunction.Coalesce = + DBFunction.Coalesce(expr, more.toVector) + + def power(base: Int, expr: SelectExpr): DBFunction = + DBFunction.Power(expr, base) + + def lit[A](value: A)(implicit P: Put[A]): SelectExpr.SelectLit[A] = + SelectExpr.SelectLit(value, None) + + def plus(expr: SelectExpr, more: SelectExpr*): DBFunction = + DBFunction.Plus(expr, more.toVector) + + def mult(expr: SelectExpr, more: SelectExpr*): DBFunction = + DBFunction.Mult(expr, more.toVector) def and(c: Condition, cs: Condition*): Condition = c match { @@ -75,6 +93,8 @@ trait DSL extends DoobieMeta { else and(c, cs: _*) implicit final class ColumnOps[A](col: Column[A]) { + def s: SelectExpr = SelectExpr.SelectColumn(col, None) + def as(alias: String) = SelectExpr.SelectColumn(col, Some(alias)) def setTo(value: A)(implicit P: Put[A]): Setter[A] = Setter.SetValue(col, value) @@ -86,10 +106,10 @@ trait DSL extends DoobieMeta { Setter.Increment(col, amount) def asc: OrderBy = - OrderBy(SelectExpr.SelectColumn(col), OrderBy.OrderType.Asc) + OrderBy(SelectExpr.SelectColumn(col, None), OrderBy.OrderType.Asc) def desc: OrderBy = - OrderBy(SelectExpr.SelectColumn(col), OrderBy.OrderType.Desc) + OrderBy(SelectExpr.SelectColumn(col, None), OrderBy.OrderType.Desc) def ===(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Eq, value) @@ -155,6 +175,38 @@ trait DSL extends DoobieMeta { not(c) } + implicit final class DBFunctionOps(dbf: DBFunction) { + def s: SelectExpr = SelectExpr.SelectFun(dbf, None) + def as(alias: String) = SelectExpr.SelectFun(dbf, Some(alias)) + + def ===[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Eq, value) + + def ====(value: String): Condition = + Condition.CompareFVal(dbf, Operator.Eq, value) + + def like[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.LowerLike, value) + + def likes(value: String): Condition = + Condition.CompareFVal(dbf, Operator.LowerLike, value) + + def <=[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Lte, value) + + def >=[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Gte, value) + + def >[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Gt, value) + + def <[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Lt, value) + + def <>[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(dbf, Operator.Neq, value) + } + } object DSL extends DSL diff --git a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala index 51250513..fffa53f9 100644 --- a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala +++ b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala @@ -6,8 +6,8 @@ object GroupBy { def apply(c: Column[_], cs: Column[_]*): GroupBy = GroupBy( - SelectExpr.SelectColumn(c), - cs.toVector.map(SelectExpr.SelectColumn.apply), + SelectExpr.SelectColumn(c, None), + cs.toVector.map(c => SelectExpr.SelectColumn(c, None)), None ) } diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index fd660332..c7076be5 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -11,11 +11,18 @@ sealed trait Select { def run: Fragment = DoobieQuery(this) - def orderBy(ob: OrderBy, obs: OrderBy*): Select = + def orderBy(ob: OrderBy, obs: OrderBy*): Select.Ordered = Select.Ordered(this, ob, obs.toVector) - def orderBy(c: Column[_]): Select = - orderBy(OrderBy(SelectExpr.SelectColumn(c), OrderBy.OrderType.Asc)) + def orderBy(c: Column[_]): Select.Ordered = + orderBy(OrderBy(SelectExpr.SelectColumn(c, None), OrderBy.OrderType.Asc)) + + def limit(n: Int): Select = + this match { + case Select.Limit(q, _) => Select.Limit(q, n) + case _ => + Select.Limit(this, n) + } } object Select { @@ -49,4 +56,6 @@ object Select { case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) extends Select + + case class Limit(q: Select, limit: Int) extends Select } diff --git a/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala index 1ccb3b90..fec6eee4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala @@ -1,11 +1,27 @@ package docspell.store.qb -sealed trait SelectExpr +import doobie.Put + +sealed trait SelectExpr { self => + def as(alias: String): SelectExpr +} object SelectExpr { - case class SelectColumn(column: Column[_]) extends SelectExpr + case class SelectColumn(column: Column[_], alias: Option[String]) extends SelectExpr { + def as(a: String): SelectColumn = + copy(alias = Some(a)) + } - case class SelectFun(fun: DBFunction) extends SelectExpr + case class SelectFun(fun: DBFunction, alias: Option[String]) extends SelectExpr { + def as(a: String): SelectFun = + copy(alias = Some(a)) + } + + case class SelectLit[A](value: A, alias: Option[String])(implicit val P: Put[A]) + extends SelectExpr { + def as(a: String): SelectLit[A] = + copy(alias = Some(a)) + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala new file mode 100644 index 00000000..8b79cfdf --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala @@ -0,0 +1,20 @@ +package docspell.store.qb.impl + +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +trait CommonBuilder { + def column(col: Column[_]): Fragment = { + val prefix = col.table.alias.getOrElse(col.table.tableName) + if (prefix.isEmpty) columnNoPrefix(col) + else Fragment.const0(prefix) ++ Fragment.const0(".") ++ Fragment.const0(col.name) + } + + def columnNoPrefix(col: Column[_]): Fragment = + Fragment.const0(col.name) + + def appendAs(alias: Option[String]): Fragment = + alias.map(a => fr" AS" ++ Fragment.const(a)).getOrElse(Fragment.empty) +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index 2dfbd8a8..8f56738a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -25,6 +25,17 @@ object ConditionBuilder { } colFrag ++ opFrag ++ valFrag + case c @ Condition.CompareFVal(dbf, op, value) => + val opFrag = operator(op) + val valFrag = buildValue(value)(c.P) + val dbfFrag = op match { + case Operator.LowerLike => + lower(dbf) + case _ => + DBFunctionBuilder.build(dbf) + } + dbfFrag ++ opFrag ++ valFrag + case Condition.CompareCol(c1, op, c2) => val (c1Frag, c2Frag) = op match { case Operator.LowerLike => @@ -36,13 +47,13 @@ object ConditionBuilder { case Condition.InSubSelect(col, subsel) => val sub = DoobieQuery(subsel) - SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ sql")" + SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ parenClose case c @ Condition.InValues(col, values, toLower) => val cfrag = if (toLower) lower(col) else SelectExprBuilder.column(col) cfrag ++ sql" IN (" ++ values.toList .map(a => buildValue(a)(c.P)) - .reduce(_ ++ comma ++ _) ++ sql")" + .reduce(_ ++ comma ++ _) ++ parenClose case Condition.IsNull(col) => SelectExprBuilder.column(col) ++ fr" is null" @@ -89,5 +100,8 @@ object ConditionBuilder { fr"$v" def lower(col: Column[_]): Fragment = - Fragment.const0("LOWER(") ++ SelectExprBuilder.column(col) ++ sql")" + Fragment.const0("LOWER(") ++ SelectExprBuilder.column(col) ++ parenClose + + def lower(dbf: DBFunction): Fragment = + Fragment.const0("LOWER(") ++ DBFunctionBuilder.build(dbf) ++ parenClose } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala new file mode 100644 index 00000000..cbc48ec8 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -0,0 +1,40 @@ +package docspell.store.qb.impl + +import docspell.store.qb.DBFunction + +import doobie._ +import doobie.implicits._ + +object DBFunctionBuilder extends CommonBuilder { + private val comma = fr"," + + def build(expr: DBFunction): Fragment = + expr match { + case DBFunction.CountAll => + sql"COUNT(*)" + + case DBFunction.Count(col) => + sql"COUNT(" ++ column(col) ++ fr")" + + case DBFunction.Max(col) => + sql"MAX(" ++ column(col) ++ fr")" + + case DBFunction.Min(col) => + sql"MIN(" ++ column(col) ++ fr")" + + case DBFunction.Coalesce(expr, exprs) => + val v = exprs.prepended(expr).map(SelectExprBuilder.build) + sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ sql")" + + case DBFunction.Power(expr, base) => + sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")" + + case DBFunction.Plus(expr, more) => + val v = more.prepended(expr).map(SelectExprBuilder.build) + v.reduce(_ ++ fr" +" ++ _) + + case DBFunction.Mult(expr, more) => + val v = more.prepended(expr).map(SelectExprBuilder.build) + v.reduce(_ ++ fr" *" ++ _) + } +} diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala index e20d9e72..b1a45d70 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala @@ -34,6 +34,8 @@ object DoobieQuery { val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _) build(distinct)(q) ++ fr"ORDER BY" ++ order + case Select.Limit(q, n) => + build(distinct)(q) ++ fr" LIMIT $n" } def buildSimple(sq: Select.SimpleSelect): Fragment = { diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala index f59d1fe5..b027b704 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala @@ -2,32 +2,21 @@ package docspell.store.qb.impl import docspell.store.qb._ -import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} -object SelectExprBuilder { +object SelectExprBuilder extends CommonBuilder { def build(expr: SelectExpr): Fragment = expr match { - case SelectExpr.SelectColumn(col) => - column(col) + case SelectExpr.SelectColumn(col, alias) => + column(col) ++ appendAs(alias) - case SelectExpr.SelectFun(DBFunction.CountAll(alias)) => - sql"COUNT(*) AS" ++ Fragment.const(alias) + case s @ SelectExpr.SelectLit(value, aliasOpt) => + ConditionBuilder.buildValue(value)(s.P) ++ appendAs(aliasOpt) - case SelectExpr.SelectFun(DBFunction.Count(col, alias)) => - sql"COUNT(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) + case SelectExpr.SelectFun(fun, alias) => + DBFunctionBuilder.build(fun) ++ appendAs(alias) - case SelectExpr.SelectFun(DBFunction.Max(col, alias)) => - sql"MAX(" ++ column(col) ++ fr") AS" ++ Fragment.const(alias) } - def column(col: Column[_]): Fragment = { - val prefix = col.table.alias.getOrElse(col.table.tableName) - if (prefix.isEmpty) columnNoPrefix(col) - else Fragment.const0(prefix) ++ Fragment.const0(".") ++ Fragment.const0(col.name) - } - - def columnNoPrefix(col: Column[_]): Fragment = - Fragment.const0(col.name) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QJob.scala b/modules/store/src/main/scala/docspell/store/queries/QJob.scala index f3521ed9..815f9de6 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QJob.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QJob.scala @@ -1,5 +1,6 @@ package docspell.store.queries +import cats.data.NonEmptyList import cats.effect.Effect import cats.implicits._ import fs2.Stream @@ -7,7 +8,8 @@ import fs2.Stream import docspell.common._ import docspell.common.syntax.all._ import docspell.store.Store -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records.{RJob, RJobGroupUse, RJobLog} import doobie._ @@ -89,70 +91,60 @@ object QJob { now: Timestamp, initialPause: Duration ): ConnectionIO[Option[Ident]] = { - val JC = RJob.Columns - val waiting: JobState = JobState.Waiting - val stuck: JobState = JobState.Stuck - val jgroup = JC.group.prefix("a") - val jstate = JC.state.prefix("a") - val ugroup = RJobGroupUse.Columns.group.prefix("b") - val uworker = RJobGroupUse.Columns.worker.prefix("b") - - val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++ - fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}" + val JC = RJob.as("a") + val G = RJobGroupUse.as("b") + val stuckTrigger = stuckTriggerValue(JC, initialPause, now) val stateCond = - or(jstate.is(waiting), and(jstate.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}")) + JC.state === JobState.waiting || (JC.state === JobState.stuck && stuckTrigger < now.toMillis) - val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++ - fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++ - fr"WHERE" ++ and(uworker.is(worker), stateCond) ++ - fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres - val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++ - fr"WHERE" ++ stateCond + val sql1 = + Select( + List(max(JC.group).as("g")), + from(JC).innerJoin(G, JC.group === G.group), + G.worker === worker && stateCond + ) - val union = - sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null" + val sql2 = + Select(List(min(JC.group).as("g")), from(JC), stateCond) - union - .query[Ident] - .to[List] - .map( - _.headOption - ) // either one or two results, but may be empty if RJob table is empty + val gcol = Column[String]("g", TableDef("")) + val groups = + Select(select(gcol), fromSubSelect(union(sql1, sql2)).as("t0"), gcol.isNull.negate) + + // either 0, one or two results, but may be empty if RJob table is empty + groups.run.query[Ident].to[List].map(_.headOption) } + private def stuckTriggerValue(t: RJob.Table, initialPause: Duration, now: Timestamp) = + plus( + coalesce(t.startedmillis.s, lit(now.toMillis)).s, + mult(power(2, t.retries.s).s, lit(initialPause.millis)).s + ) + def selectNextJob( group: Ident, prio: Priority, initialPause: Duration, now: Timestamp ): ConnectionIO[Option[RJob]] = { - val JC = RJob.Columns + val JC = RJob.T val psort = if (prio == Priority.High) JC.priority.desc else JC.priority.asc - val waiting: JobState = JobState.Waiting - val stuck: JobState = JobState.Stuck + val waiting = JobState.waiting + val stuck = JobState.stuck - val stuckTrigger = - coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2( - JC.retries - ) ++ fr"* ${initialPause.millis}" - val sql = selectSimple( - JC.all, - RJob.table, - and( - JC.group.is(group), - or( - JC.state.is(waiting), - and(JC.state.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}") - ) - ) - ) ++ - orderBy(JC.state.asc, psort, JC.submitted.asc) ++ - fr"LIMIT 1" + val stuckTrigger = stuckTriggerValue(JC, initialPause, now) + val sql = + Select( + select(JC.all), + from(JC), + JC.group === group && (JC.state === waiting || + (JC.state === stuck && stuckTrigger < now.toMillis)) + ).orderBy(JC.state.asc, psort, JC.submitted.asc).limit(1) - sql.query[RJob].option + sql.run.query[RJob].option } def setCancelled[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = @@ -212,39 +204,34 @@ object QJob { collective: Ident, max: Long ): Stream[ConnectionIO, (RJob, Vector[RJobLog])] = { - val JC = RJob.Columns - val waiting: Set[JobState] = Set(JobState.Waiting, JobState.Stuck, JobState.Scheduled) - val running: Set[JobState] = Set(JobState.Running) - val done = JobState.all.diff(waiting).diff(running) + val JC = RJob.T + val waiting = NonEmptyList.of(JobState.Waiting, JobState.Stuck, JobState.Scheduled) + val running = NonEmptyList.of(JobState.Running) + //val done = JobState.all.filterNot(js => ).diff(waiting).diff(running) def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = { val refDate = now.minusHours(24) + val runningJobs = Select( + select(JC.all), + from(JC), + JC.group === collective && JC.state.in(running) + ).orderBy(JC.submitted.desc).run.query[RJob].stream - val runningJobs = (selectSimple( - JC.all, - RJob.table, - and(JC.group.is(collective), JC.state.isOneOf(running.toSeq)) - ) ++ orderBy(JC.submitted.desc)).query[RJob].stream + val waitingJobs = Select( + select(JC.all), + from(JC), + JC.group === collective && JC.state.in(waiting) && JC.submitted > refDate + ).orderBy(JC.submitted.desc).run.query[RJob].stream.take(max) - val waitingJobs = (selectSimple( - JC.all, - RJob.table, + val doneJobs = Select( + select(JC.all), + from(JC), and( - JC.group.is(collective), - JC.state.isOneOf(waiting.toSeq), - JC.submitted.isGt(refDate) + JC.group === collective, + JC.state.in(JobState.done), + JC.submitted > refDate ) - ) ++ orderBy(JC.submitted.desc)).query[RJob].stream.take(max) - - val doneJobs = (selectSimple( - JC.all, - RJob.table, - and( - JC.group.is(collective), - JC.state.isOneOf(done.toSeq), - JC.submitted.isGt(refDate) - ) - ) ++ orderBy(JC.submitted.desc)).query[RJob].stream.take(max) + ).orderBy(JC.submitted.desc).run.query[RJob].stream.take(max) runningJobs ++ waitingJobs ++ doneJobs } diff --git a/modules/store/src/main/scala/docspell/store/records/RJob.scala b/modules/store/src/main/scala/docspell/store/records/RJob.scala index 0bc2dc14..5f7b8850 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJob.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJob.scala @@ -1,12 +1,12 @@ package docspell.store.records -import cats.effect.Sync +import cats.data.NonEmptyList import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -34,7 +34,7 @@ case class RJob( s"${id.id.substring(0, 9)}.../${group.id}/${task.id}/$priority" def isFinalState: Boolean = - JobState.done.contains(state) + JobState.done.toList.contains(state) def isInProgress: Boolean = JobState.inProgress.contains(state) @@ -71,25 +71,25 @@ object RJob { None ) - val table = fr"job" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "job" - object Columns { - val id = Column("jid") - val task = Column("task") - val group = Column("group_") - val args = Column("args") - val subject = Column("subject") - val submitted = Column("submitted") - val submitter = Column("submitter") - val priority = Column("priority") - val state = Column("state") - val retries = Column("retries") - val progress = Column("progress") - val tracker = Column("tracker") - val worker = Column("worker") - val started = Column("started") - val startedmillis = Column("startedmillis") - val finished = Column("finished") + val id = Column[Ident]("jid", this) + val task = Column[Ident]("task", this) + val group = Column[Ident]("group_", this) + val args = Column[String]("args", this) + val subject = Column[String]("subject", this) + val submitted = Column[Timestamp]("submitted", this) + val submitter = Column[Ident]("submitter", this) + val priority = Column[Priority]("priority", this) + val state = Column[JobState]("state", this) + val retries = Column[Int]("retries", this) + val progress = Column[Int]("progress", this) + val tracker = Column[Ident]("tracker", this) + val worker = Column[Ident]("worker", this) + val started = Column[Timestamp]("started", this) + val startedmillis = Column[Long]("startedmillis", this) + val finished = Column[Timestamp]("finished", this) val all = List( id, task, @@ -109,163 +109,174 @@ object RJob { ) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RJob): ConnectionIO[Int] = { val smillis = v.started.map(_.toMillis) - val sql = insertRow( - table, - all ++ List(startedmillis), + DML.insert( + T, + T.all ++ List(T.startedmillis), fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis" ) - sql.update.run } def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] = - if (ids.isEmpty) Sync[ConnectionIO].pure(Vector.empty[RJob]) - else selectSimple(all, table, id.isOneOf(ids)).query[RJob].to[Vector] + NonEmptyList.fromList(ids.toList) match { + case None => + Vector.empty[RJob].pure[ConnectionIO] + case Some(nel) => + run(select(T.all), from(T), T.id.in(nel)).query[RJob].to[Vector] + } def findByIdAndGroup(jobId: Ident, jobGroup: Ident): ConnectionIO[Option[RJob]] = - selectSimple(all, table, and(id.is(jobId), group.is(jobGroup))).query[RJob].option + run(select(T.all), from(T), T.id === jobId && T.group === jobGroup).query[RJob].option def findById(jobId: Ident): ConnectionIO[Option[RJob]] = - selectSimple(all, table, id.is(jobId)).query[RJob].option + run(select(T.all), from(T), T.id === jobId).query[RJob].option def findByIdAndWorker(jobId: Ident, workerId: Ident): ConnectionIO[Option[RJob]] = - selectSimple(all, table, and(id.is(jobId), worker.is(workerId))).query[RJob].option + run(select(T.all), from(T), T.id === jobId && T.worker === workerId) + .query[RJob] + .option def setRunningToWaiting(workerId: Ident): ConnectionIO[Int] = { - val states: Seq[JobState] = List(JobState.Running, JobState.Scheduled) - updateRow( - table, - and(worker.is(workerId), state.isOneOf(states)), - state.setTo(JobState.Waiting: JobState) - ).update.run + val states: NonEmptyList[JobState] = + NonEmptyList.of(JobState.Running, JobState.Scheduled) + DML.update( + T, + where(T.worker === workerId, T.state.in(states)), + DML.set(T.state.setTo(JobState.waiting)) + ) } def incrementRetries(jobid: Ident): ConnectionIO[Int] = - updateRow( - table, - and(id.is(jobid), state.is(JobState.Stuck: JobState)), - retries.f ++ fr"=" ++ retries.f ++ fr"+ 1" - ).update.run + DML + .update( + T, + where(T.id === jobid, T.state === JobState.stuck), + DML.set(T.retries.increment(1)) + ) def setRunning(jobId: Ident, workerId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Running: JobState), - started.setTo(now), - startedmillis.setTo(now.toMillis), - worker.setTo(workerId) + DML.update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.running), + T.started.setTo(now), + T.startedmillis.setTo(now.toMillis), + T.worker.setTo(workerId) ) - ).update.run + ) def setWaiting(jobId: Ident): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Waiting: JobState), - started.setTo(None: Option[Timestamp]), - startedmillis.setTo(None: Option[Long]), - finished.setTo(None: Option[Timestamp]) + DML + .update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.Waiting: JobState), + T.started.setTo(None: Option[Timestamp]), + T.startedmillis.setTo(None: Option[Long]), + T.finished.setTo(None: Option[Timestamp]) + ) ) - ).update.run def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] = for { _ <- incrementRetries(jobId) - n <- updateRow( - table, - and( - id.is(jobId), - or(worker.isNull, worker.is(workerId)), - state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck)) + n <- DML.update( + T, + where( + T.id === jobId, + or(T.worker.isNull, T.worker === workerId), + T.state.in(NonEmptyList.of(JobState.waiting, JobState.stuck)) ), - commas( - state.setTo(JobState.Scheduled: JobState), - worker.setTo(workerId) + DML.set( + T.state.setTo(JobState.scheduled), + T.worker.setTo(workerId) ) - ).update.run + ) } yield n def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Success: JobState), - finished.setTo(now) + DML + .update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.success), + T.finished.setTo(now) + ) ) - ).update.run def setStuck(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Stuck: JobState), - finished.setTo(now) + DML.update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.stuck), + T.finished.setTo(now) ) - ).update.run + ) def setFailed(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Failed: JobState), - finished.setTo(now) + DML.update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.failed), + T.finished.setTo(now) ) - ).update.run + ) def setCancelled(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow( - table, - id.is(jobId), - commas( - state.setTo(JobState.Cancelled: JobState), - finished.setTo(now) + DML.update( + T, + T.id === jobId, + DML.set( + T.state.setTo(JobState.cancelled), + T.finished.setTo(now) ) - ).update.run + ) def setPriority(jobId: Ident, jobGroup: Ident, prio: Priority): ConnectionIO[Int] = - updateRow( - table, - and(id.is(jobId), group.is(jobGroup), state.is(JobState.waiting)), - priority.setTo(prio) - ).update.run + DML.update( + T, + where(T.id === jobId, T.group === jobGroup, T.state === JobState.waiting), + DML.set(T.priority.setTo(prio)) + ) def getRetries(jobId: Ident): ConnectionIO[Option[Int]] = - selectSimple(List(retries), table, id.is(jobId)).query[Int].option + run(select(T.retries), from(T), T.id === jobId).query[Int].option def setProgress(jobId: Ident, perc: Int): ConnectionIO[Int] = - updateRow(table, id.is(jobId), progress.setTo(perc)).update.run + DML.update(T, T.id === jobId, DML.set(T.progress.setTo(perc))) def selectWaiting: ConnectionIO[Option[RJob]] = { - val sql = selectSimple(all, table, state.is(JobState.Waiting: JobState)) + val sql = run(select(T.all), from(T), T.state === JobState.waiting) sql.query[RJob].to[Vector].map(_.headOption) } - def selectGroupInState(states: Seq[JobState]): ConnectionIO[Vector[Ident]] = { + def selectGroupInState(states: NonEmptyList[JobState]): ConnectionIO[Vector[Ident]] = { val sql = - selectDistinct(List(group), table, state.isOneOf(states)) ++ orderBy(group.f) - sql.query[Ident].to[Vector] + Select(select(T.group), from(T), T.state.in(states)).orderBy(T.group) + sql.run.query[Ident].to[Vector] } def delete(jobId: Ident): ConnectionIO[Int] = for { n0 <- RJobLog.deleteAll(jobId) - n1 <- deleteFrom(table, id.is(jobId)).update.run + n1 <- DML.delete(T, T.id === jobId) } yield n0 + n1 def findIdsDoneAndOlderThan(ts: Timestamp): Stream[ConnectionIO, Ident] = - selectSimple( - Seq(id), - table, - and(state.isOneOf(JobState.done.toSeq), or(finished.isNull, finished.isLt(ts))) + run( + select(T.id), + from(T), + T.state.in(JobState.done) && (T.finished.isNull || T.finished < ts) ).query[Ident].stream def deleteDoneAndOlderThan(ts: Timestamp, batch: Int): ConnectionIO[Int] = @@ -277,10 +288,10 @@ object RJob { .foldMonoid def findNonFinalByTracker(trackerId: Ident): ConnectionIO[Option[RJob]] = - selectSimple( - all, - table, - and(tracker.is(trackerId), state.isOneOf(JobState.all.diff(JobState.done).toSeq)) + run( + select(T.all), + from(T), + where(T.tracker === trackerId, T.state.in(JobState.notDone)) ).query[RJob].option } diff --git a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala index 1ce0e448..9cf4aec4 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala @@ -3,8 +3,8 @@ package docspell.store.records import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -12,25 +12,27 @@ import doobie.implicits._ case class RJobGroupUse(groupId: Ident, workerId: Ident) {} object RJobGroupUse { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "jobgroupuse" - val table = fr"jobgroupuse" - - object Columns { - val group = Column("groupid") - val worker = Column("workerid") + val group = Column[Ident]("groupid", this) + val worker = Column[Ident]("workerid", this) val all = List(group, worker) } - import Columns._ + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RJobGroupUse): ConnectionIO[Int] = - insertRow(table, all, fr"${v.groupId},${v.workerId}").update.run + DML.insert(T, T.all, fr"${v.groupId},${v.workerId}") def updateGroup(v: RJobGroupUse): ConnectionIO[Int] = - updateRow(table, worker.is(v.workerId), group.setTo(v.groupId)).update.run + DML.update(T, T.worker === v.workerId, DML.set(T.group.setTo(v.groupId))) def setGroup(v: RJobGroupUse): ConnectionIO[Int] = updateGroup(v).flatMap(n => if (n > 0) n.pure[ConnectionIO] else insert(v)) def findGroup(workerId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(group), table, worker.is(workerId)).query[Ident].option + run(select(T.group), from(T), T.worker === workerId).query[Ident].option } diff --git a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala index 546aa5fe..999e9570 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala @@ -1,8 +1,8 @@ package docspell.store.records import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -16,35 +16,39 @@ case class RJobLog( ) {} object RJobLog { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "joblog" - val table = fr"joblog" - - object Columns { - val id = Column("id") - val jobId = Column("jid") - val level = Column("level") - val created = Column("created") - val message = Column("message") + val id = Column[Ident]("id", this) + val jobId = Column[Ident]("jid", this) + val level = Column[LogLevel]("level", this) + val created = Column[Timestamp]("created", this) + val message = Column[String]("message", this) val all = List(id, jobId, level, created, message) // separate column only for sorting, so not included in `all` and // the case class - val counter = Column("counter") + val counter = Column[Long]("counter", this) } - import Columns._ + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RJobLog): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}" - ).update.run + ) def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] = - (selectSimple(all, table, jobId.is(id)) ++ orderBy(created.asc, counter.asc)) + Select(select(T.all), from(T), T.jobId === id) + .orderBy(T.created.asc, T.counter.asc) + .run .query[RJobLog] .to[Vector] def deleteAll(job: Ident): ConnectionIO[Int] = - deleteFrom(table, jobId.is(job)).update.run + DML.delete(T, T.jobId === job) } From 87eb8c7f55d8c02696707293aeb22ff10feeb574 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sat, 12 Dec 2020 12:23:01 +0100 Subject: [PATCH 11/38] Convert more records --- .../store/records/RFtsMigration.scala | 40 +++++++++++-------- .../docspell/store/records/RInvitation.scala | 28 +++++++------ 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala b/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala index b02d26f7..b2f21930 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala @@ -4,8 +4,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -30,32 +30,38 @@ object RFtsMigration { now <- Timestamp.current[F] } yield RFtsMigration(newId, version, ftsEngine, description, now) - val table = fr"fts_migration" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "fts_migration" - object Columns { - val id = Column("id") - val version = Column("version") - val ftsEngine = Column("fts_engine") - val description = Column("description") - val created = Column("created") + val id = Column[Ident]("id", this) + val version = Column[Int]("version", this) + val ftsEngine = Column[Ident]("fts_engine", this) + val description = Column[String]("description", this) + val created = Column[Timestamp]("created", this) val all = List(id, version, ftsEngine, description, created) } - import Columns._ + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def insert(v: RFtsMigration): ConnectionIO[Int] = - insertRow( - table, - all, - fr"${v.id},${v.version},${v.ftsEngine},${v.description},${v.created}" - ).updateWithLogHandler(LogHandler.nop).run + DML + .insertFragment( + T, + T.all, + Seq(fr"${v.id},${v.version},${v.ftsEngine},${v.description},${v.created}") + ) + .updateWithLogHandler(LogHandler.nop) + .run def exists(vers: Int, engine: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(version.is(vers), ftsEngine.is(engine))) + run(select(count(T.id)), from(T), T.version === vers && T.ftsEngine === engine) .query[Int] .unique .map(_ > 0) def deleteById(rId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(rId)).update.run + DML.delete(T, T.id === rId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala index 6a516788..f3243566 100644 --- a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala +++ b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala @@ -4,8 +4,8 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -13,15 +13,17 @@ import doobie.implicits._ case class RInvitation(id: Ident, created: Timestamp) {} object RInvitation { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "invitation" - val table = fr"invitation" - - object Columns { - val id = Column("id") - val created = Column("created") + val id = Column[Ident]("id", this) + val created = Column[Timestamp]("created", this) val all = List(id, created) } - import Columns._ + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) def generate[F[_]: Sync]: F[RInvitation] = for { @@ -30,19 +32,19 @@ object RInvitation { } yield RInvitation(i, c) def insert(v: RInvitation): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.created}").update.run + DML.insert(T, T.all, fr"${v.id},${v.created}") def insertNew: ConnectionIO[RInvitation] = generate[ConnectionIO].flatMap(v => insert(v).map(_ => v)) def findById(invite: Ident): ConnectionIO[Option[RInvitation]] = - selectSimple(all, table, id.is(invite)).query[RInvitation].option + run(select(T.all), from(T), T.id === invite).query[RInvitation].option def delete(invite: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(invite)).update.run + DML.delete(T, T.id === invite) def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = { - val get = selectCount(id, table, and(id.is(invite), created.isGt(minCreated))) + val get = run(select(count(T.id)), from(T), T.id === invite && T.created > minCreated) .query[Int] .unique for { @@ -52,5 +54,5 @@ object RInvitation { } def deleteOlderThan(ts: Timestamp): ConnectionIO[Int] = - deleteFrom(table, created.isLt(ts)).update.run + DML.delete(T, T.created < ts) } From d6f28d3eca4a1c0d8777bc17381517de871d4acf Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 00:26:58 +0100 Subject: [PATCH 12/38] Convert folder --- .../main/scala/docspell/store/qb/Column.scala | 11 +- .../scala/docspell/store/qb/Condition.scala | 8 +- .../scala/docspell/store/qb/CteBind.scala | 9 + .../scala/docspell/store/qb/DBFunction.scala | 9 +- .../main/scala/docspell/store/qb/DML.scala | 2 +- .../main/scala/docspell/store/qb/DSL.scala | 48 +++-- .../main/scala/docspell/store/qb/Select.scala | 29 +-- .../scala/docspell/store/qb/SelectExpr.scala | 12 +- .../scala/docspell/store/qb/TableDef.scala | 11 +- .../store/qb/impl/ConditionBuilder.scala | 6 +- .../store/qb/impl/DBFunctionBuilder.scala | 20 +- .../store/qb/impl/FromExprBuilder.scala | 2 +- ...{DoobieQuery.scala => SelectBuilder.scala} | 26 +-- .../store/qb/impl/SelectExprBuilder.scala | 8 +- .../docspell/store/queries/QFolder.scala | 172 ++++++++---------- .../scala/docspell/store/queries/QItem.scala | 28 ++- .../docspell/store/records/RContact.scala | 2 +- .../docspell/store/records/RFolder.scala | 61 +++---- .../store/records/RFolderMember.scala | 37 ++-- .../scala/docspell/store/records/RTag.scala | 2 +- .../store/qb/impl/DoobieQueryTest.scala | 2 +- 21 files changed, 295 insertions(+), 210 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/qb/CteBind.scala rename modules/store/src/main/scala/docspell/store/qb/impl/{DoobieQuery.scala => SelectBuilder.scala} (71%) diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala index a8465417..60d49abc 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Column.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -1,5 +1,14 @@ package docspell.store.qb -case class Column[A](name: String, table: TableDef) +case class Column[A](name: String, table: TableDef) { + def inTable(t: TableDef): Column[A] = + copy(table = t) + + def s: SelectExpr = + SelectExpr.SelectColumn(this, None) + + def as(alias: String): SelectExpr = + SelectExpr.SelectColumn(this, Some(alias)) +} object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 4738545a..264889f7 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -4,7 +4,13 @@ import cats.data.NonEmptyList import doobie._ -sealed trait Condition {} +sealed trait Condition { + def s: SelectExpr.SelectCondition = + SelectExpr.SelectCondition(this, None) + + def as(alias: String): SelectExpr.SelectCondition = + SelectExpr.SelectCondition(this, Some(alias)) +} object Condition { diff --git a/modules/store/src/main/scala/docspell/store/qb/CteBind.scala b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala new file mode 100644 index 00000000..0a22a056 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala @@ -0,0 +1,9 @@ +package docspell.store.qb + +case class CteBind(name: TableDef, select: Select) {} + +object CteBind { + + def apply(t: (TableDef, Select)): CteBind = + CteBind(t._1, t._2) +} diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 52b50024..24de3520 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -21,7 +21,12 @@ object DBFunction { case class Power(expr: SelectExpr, base: Int) extends DBFunction - case class Plus(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction + case class Calc(op: Operator, left: SelectExpr, right: SelectExpr) extends DBFunction - case class Mult(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction + sealed trait Operator + object Operator { + case object Plus extends Operator + case object Minus extends Operator + case object Mult extends Operator + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/DML.scala b/modules/store/src/main/scala/docspell/store/qb/DML.scala index 194ee3db..e8fd6dcf 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DML.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DML.scala @@ -48,7 +48,7 @@ object DML { cond: Option[Condition], setter: Seq[Setter[_]] ): Fragment = { - val condFrag = cond.map(DoobieQuery.cond).getOrElse(Fragment.empty) + val condFrag = cond.map(SelectBuilder.cond).getOrElse(Fragment.empty) fr"UPDATE" ++ FromExprBuilder.buildTable(table) ++ fr"SET" ++ setter .map(s => buildSetter(s)) diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 849305c4..6223ebc0 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -3,30 +3,39 @@ package docspell.store.qb import cats.data.NonEmptyList import docspell.store.impl.DoobieMeta -import docspell.store.qb.impl.DoobieQuery +import docspell.store.qb.impl.SelectBuilder import doobie.{Fragment, Put} trait DSL extends DoobieMeta { def run(projection: Seq[SelectExpr], from: FromExpr): Fragment = - DoobieQuery(Select(projection, from, None)) + SelectBuilder(Select(projection, from)) def run(projection: Seq[SelectExpr], from: FromExpr, where: Condition): Fragment = - DoobieQuery(Select(projection, from, where)) + SelectBuilder(Select(projection, from, where)) def runDistinct( projection: Seq[SelectExpr], from: FromExpr, where: Condition ): Fragment = - DoobieQuery.distinct(Select(projection, from, where)) + SelectBuilder(Select(projection, from, where).distinct) + + def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl = + DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector) + + def select(cond: Condition): Seq[SelectExpr] = + Seq(SelectExpr.SelectCondition(cond, None)) def select(dbf: DBFunction): Seq[SelectExpr] = Seq(SelectExpr.SelectFun(dbf, None)) + def select(e: SelectExpr, es: SelectExpr*): Seq[SelectExpr] = + es.prepended(e) + def select(c: Column[_], cs: Column[_]*): Seq[SelectExpr] = - select(c :: cs.toList) + cs.prepended(c).map(col => SelectExpr.SelectColumn(col, None)) def select(seq: Seq[Column[_]], seqs: Seq[Column[_]]*): Seq[SelectExpr] = (seq ++ seqs.flatten).map(c => SelectExpr.SelectColumn(c, None)) @@ -43,6 +52,9 @@ trait DSL extends DoobieMeta { def count(c: Column[_]): DBFunction = DBFunction.Count(c) + def countAll: DBFunction = + DBFunction.CountAll + def max(c: Column[_]): DBFunction = DBFunction.Max(c) @@ -58,11 +70,11 @@ trait DSL extends DoobieMeta { def lit[A](value: A)(implicit P: Put[A]): SelectExpr.SelectLit[A] = SelectExpr.SelectLit(value, None) - def plus(expr: SelectExpr, more: SelectExpr*): DBFunction = - DBFunction.Plus(expr, more.toVector) + def plus(left: SelectExpr, right: SelectExpr): DBFunction = + DBFunction.Calc(DBFunction.Operator.Plus, left, right) - def mult(expr: SelectExpr, more: SelectExpr*): DBFunction = - DBFunction.Mult(expr, more.toVector) + def mult(left: SelectExpr, right: SelectExpr): DBFunction = + DBFunction.Calc(DBFunction.Operator.Mult, left, right) def and(c: Condition, cs: Condition*): Condition = c match { @@ -205,8 +217,22 @@ trait DSL extends DoobieMeta { def <>[A](value: A)(implicit P: Put[A]): Condition = Condition.CompareFVal(dbf, Operator.Neq, value) + + def -[A](value: A)(implicit P: Put[A]): DBFunction = + DBFunction.Calc( + DBFunction.Operator.Minus, + SelectExpr.SelectFun(dbf, None), + SelectExpr.SelectLit(value, None) + ) + } +} + +object DSL extends DSL { + + final case class WithCteDsl(cte: CteBind, ctes: Vector[CteBind]) { + + def select(s: Select): Select.WithCte = + Select.WithCte(cte, ctes, s) } } - -object DSL extends DSL diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index c7076be5..ac6d0ca1 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -1,15 +1,15 @@ package docspell.store.qb -import docspell.store.qb.impl.DoobieQuery +import docspell.store.qb.impl.SelectBuilder import doobie._ sealed trait Select { - def distinct: Fragment = - DoobieQuery.distinct(this) - def run: Fragment = - DoobieQuery(this) + SelectBuilder(this) + + def as(alias: String): SelectExpr.SelectQuery = + SelectExpr.SelectQuery(this, Some(alias)) def orderBy(ob: OrderBy, obs: OrderBy*): Select.Ordered = Select.Ordered(this, ob, obs.toVector) @@ -20,27 +20,29 @@ sealed trait Select { def limit(n: Int): Select = this match { case Select.Limit(q, _) => Select.Limit(q, n) - case _ => - Select.Limit(this, n) + case _ => Select.Limit(this, n) } } object Select { + def apply(projection: Seq[SelectExpr], from: FromExpr) = + SimpleSelect(false, projection, from, None, None) def apply( projection: Seq[SelectExpr], from: FromExpr, where: Condition - ) = SimpleSelect(projection, from, Some(where), None) + ) = SimpleSelect(false, projection, from, Some(where), None) def apply( projection: Seq[SelectExpr], from: FromExpr, - where: Option[Condition] = None, - groupBy: Option[GroupBy] = None - ) = SimpleSelect(projection, from, where, groupBy) + where: Condition, + groupBy: GroupBy + ) = SimpleSelect(false, projection, from, Some(where), Some(groupBy)) case class SimpleSelect( + distinctFlag: Boolean, projection: Seq[SelectExpr], from: FromExpr, where: Option[Condition], @@ -48,6 +50,9 @@ object Select { ) extends Select { def group(gb: GroupBy): SimpleSelect = copy(groupBy = Some(gb)) + + def distinct: SimpleSelect = + copy(distinctFlag = true) } case class Union(q: Select, qs: Vector[Select]) extends Select @@ -58,4 +63,6 @@ object Select { extends Select case class Limit(q: Select, limit: Int) extends Select + + case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select } diff --git a/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala index fec6eee4..ba029f5c 100644 --- a/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala @@ -2,7 +2,7 @@ package docspell.store.qb import doobie.Put -sealed trait SelectExpr { self => +sealed trait SelectExpr { def as(alias: String): SelectExpr } @@ -24,4 +24,14 @@ object SelectExpr { copy(alias = Some(a)) } + case class SelectQuery(query: Select, alias: Option[String]) extends SelectExpr { + def as(a: String): SelectQuery = + copy(alias = Some(a)) + } + + case class SelectCondition(cond: Condition, alias: Option[String]) extends SelectExpr { + def as(a: String): SelectCondition = + copy(alias = Some(a)) + } + } diff --git a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala index 072d98c5..4ef6cfa4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala +++ b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala @@ -9,8 +9,11 @@ trait TableDef { object TableDef { def apply(table: String, aliasName: Option[String] = None): TableDef = - new TableDef { - def tableName: String = table - def alias: Option[String] = aliasName - } + BasicTable(table, aliasName) + + final case class BasicTable(tableName: String, alias: Option[String]) extends TableDef { + def as(alias: String): BasicTable = + copy(alias = Some(alias)) + } + } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index 8f56738a..cc700b18 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -6,8 +6,8 @@ import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} object ConditionBuilder { - val or = fr"OR" - val and = fr"AND" + val or = fr" OR" + val and = fr" AND" val comma = fr"," val parenOpen = Fragment.const0("(") val parenClose = Fragment.const0(")") @@ -46,7 +46,7 @@ object ConditionBuilder { c1Frag ++ operator(op) ++ c2Frag case Condition.InSubSelect(col, subsel) => - val sub = DoobieQuery(subsel) + val sub = SelectBuilder(subsel) SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ parenClose case c @ Condition.InValues(col, values, toLower) => diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index cbc48ec8..494ec66c 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -29,12 +29,20 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Power(expr, base) => sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")" - case DBFunction.Plus(expr, more) => - val v = more.prepended(expr).map(SelectExprBuilder.build) - v.reduce(_ ++ fr" +" ++ _) + case DBFunction.Calc(op, left, right) => + SelectExprBuilder.build(left) ++ + buildOperator(op) ++ + SelectExprBuilder.build(right) - case DBFunction.Mult(expr, more) => - val v = more.prepended(expr).map(SelectExprBuilder.build) - v.reduce(_ ++ fr" *" ++ _) + } + + def buildOperator(op: DBFunction.Operator): Fragment = + op match { + case DBFunction.Operator.Minus => + fr" -" + case DBFunction.Operator.Plus => + fr" +" + case DBFunction.Operator.Mult => + fr" *" } } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala index 71ee3606..926ed330 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala @@ -17,7 +17,7 @@ object FromExprBuilder { joins.map(buildJoin).foldLeft(Fragment.empty)(_ ++ _) case FromExpr.SubSelect(sel, name) => - sql" FROM (" ++ DoobieQuery(sel) ++ fr") AS" ++ Fragment.const(name) + sql" FROM (" ++ SelectBuilder(sel) ++ fr") AS" ++ Fragment.const(name) } def buildTable(table: TableDef): Fragment = diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala similarity index 71% rename from modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala rename to modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index b1a45d70..3d64c67b 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DoobieQuery.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -5,7 +5,7 @@ import docspell.store.qb._ import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} -object DoobieQuery { +object SelectBuilder { val comma = fr"," val asc = fr" ASC" val desc = fr" DESC" @@ -13,29 +13,30 @@ object DoobieQuery { val union = fr"UNION ALL" def apply(q: Select): Fragment = - build(false)(q) + build(q) - def distinct(q: Select): Fragment = - build(true)(q) - - def build(distinct: Boolean)(q: Select): Fragment = + def build(q: Select): Fragment = q match { case sq: Select.SimpleSelect => - val sel = if (distinct) fr"SELECT DISTINCT" else fr"SELECT" + val sel = if (sq.distinctFlag) fr"SELECT DISTINCT" else fr"SELECT" sel ++ buildSimple(sq) case Select.Union(q, qs) => - qs.prepended(q).map(build(false)).reduce(_ ++ union ++ _) + qs.prepended(q).map(build).reduce(_ ++ union ++ _) case Select.Intersect(q, qs) => - qs.prepended(q).map(build(false)).reduce(_ ++ intersect ++ _) + qs.prepended(q).map(build).reduce(_ ++ intersect ++ _) case Select.Ordered(q, ob, obs) => val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _) - build(distinct)(q) ++ fr"ORDER BY" ++ order + build(q) ++ fr" ORDER BY" ++ order case Select.Limit(q, n) => - build(distinct)(q) ++ fr" LIMIT $n" + build(q) ++ fr" LIMIT $n" + + case Select.WithCte(cte, moreCte, query) => + val ctes = moreCte.prepended(cte) + fr"WITH" ++ ctes.map(buildCte).reduce(_ ++ comma ++ _) ++ fr" " ++ build(query) } def buildSimple(sq: Select.SimpleSelect): Fragment = { @@ -71,4 +72,7 @@ object DoobieQuery { val f1 = gb.having.map(cond).getOrElse(Fragment.empty) fr"GROUP BY" ++ f0 ++ f1 } + + def buildCte(bind: CteBind): Fragment = + Fragment.const(bind.name.tableName) ++ sql"AS (" ++ build(bind.select) ++ sql")" } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala index b027b704..c3b5daab 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectExprBuilder.scala @@ -2,7 +2,8 @@ package docspell.store.qb.impl import docspell.store.qb._ -import _root_.doobie.{Query => _, _} +import doobie._ +import doobie.implicits._ object SelectExprBuilder extends CommonBuilder { @@ -17,6 +18,11 @@ object SelectExprBuilder extends CommonBuilder { case SelectExpr.SelectFun(fun, alias) => DBFunctionBuilder.build(fun) ++ appendAs(alias) + case SelectExpr.SelectQuery(query, alias) => + sql"(" ++ SelectBuilder.build(query) ++ sql")" ++ appendAs(alias) + + case SelectExpr.SelectCondition(cond, alias) => + sql"(" ++ ConditionBuilder.build(cond) ++ sql")" ++ appendAs(alias) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala index 2f71fe0d..79b6341f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala @@ -4,7 +4,8 @@ import cats.data.OptionT import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ @@ -136,22 +137,16 @@ object QFolder { } def findById(id: Ident, account: AccountId): ConnectionIO[Option[FolderDetail]] = { - val user = RUser.as("u") - val mUserId = RFolderMember.Columns.user.prefix("m") - val mFolderId = RFolderMember.Columns.folder.prefix("m") - val uId = user.uid.column - val uLogin = user.login.column - val sColl = RFolder.Columns.collective.prefix("s") - val sId = RFolder.Columns.id.prefix("s") + val user = RUser.as("u") + val member = RFolderMember.as("m") + val folder = RFolder.as("s") - val from = RFolderMember.table ++ fr"m INNER JOIN" ++ - Fragment.const(user.tableName) ++ fr"u ON" ++ mUserId.is(uId) ++ fr"INNER JOIN" ++ - RFolder.table ++ fr"s ON" ++ mFolderId.is(sId) - - val memberQ = selectSimple( - Seq(uId, uLogin), - from, - and(mFolderId.is(id), sColl.is(account.collective)) + val memberQ = run( + select(user.uid, user.login), + from(member) + .innerJoin(user, member.user === user.uid) + .innerJoin(folder, member.folder === folder.id), + member.folder === id && folder.collective === account.collective ).query[IdRef].to[Vector] (for { @@ -188,92 +183,85 @@ object QFolder { // inner join user_ u on u.uid = s.owner // where s.cid = 'eike'; - val user = RUser.as("u") - val uId = user.uid.column - val uLogin = user.login.column - val sId = RFolder.Columns.id.prefix("s") - val sOwner = RFolder.Columns.owner.prefix("s") - val sName = RFolder.Columns.name.prefix("s") - val sColl = RFolder.Columns.collective.prefix("s") - val mUser = RFolderMember.Columns.user.prefix("m") - val mFolder = RFolderMember.Columns.folder.prefix("m") + val user = RUser.as("u") + val member = RFolderMember.as("m") + val folder = RFolder.as("s") + val memlogin = TableDef("memberlogin") + val memloginFolder = member.folder.inTable(memlogin) + val memloginLogn = user.login.inTable(memlogin) - //CTE - val cte: Fragment = { - val from1 = RFolderMember.table ++ fr"m INNER JOIN" ++ - Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(mUser) ++ fr"INNER JOIN" ++ - RFolder.table ++ fr"s ON" ++ sId.is(mFolder) - - val from2 = RFolder.table ++ fr"s INNER JOIN" ++ - Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(sOwner) - - withCTE( - "memberlogin" -> - (selectSimple(Seq(mFolder, uLogin), from1, sColl.is(account.collective)) ++ - fr"UNION ALL" ++ - selectSimple(Seq(sId, uLogin), from2, sColl.is(account.collective))) + val sql = + withCte( + memlogin -> union( + Select( + select(member.folder, user.login), + from(member) + .innerJoin(user, user.uid === member.user) + .innerJoin(folder, folder.id === member.folder), + folder.collective === account.collective + ), + Select( + select(folder.id, user.login), + from(folder) + .innerJoin(user, user.uid === folder.owner), + folder.collective === account.collective + ) + ) ) - } + .select( + Select( + select( + folder.id.s, + folder.name.s, + folder.owner.s, + user.login.s, + folder.created.s, + Select( + select(countAll > 0), + from(memlogin), + memloginFolder === folder.id && memloginLogn === account.user + ).as("member"), + Select( + select(countAll - 1), + from(memlogin), + memloginFolder === folder.id + ).as("member_count") + ), + from(folder) + .innerJoin(user, user.uid === folder.owner), + where( + folder.collective === account.collective &&? + idQ.map(id => folder.id === id) &&? + nameQ.map(q => folder.name.like(s"%${q.toLowerCase}%")) &&? + ownerLogin.map(login => user.login === login) + ) + ).orderBy(folder.name.asc) + ) - val isMember = - fr"SELECT COUNT(*) > 0 FROM memberlogin WHERE" ++ mFolder.prefix("").is(sId) ++ - fr"AND" ++ uLogin.prefix("").is(account.user) - - val memberCount = - fr"SELECT COUNT(*) - 1 FROM memberlogin WHERE" ++ mFolder.prefix("").is(sId) - - //Query - val cols = Seq( - sId.f, - sName.f, - sOwner.f, - uLogin.f, - RFolder.Columns.created.prefix("s").f, - fr"(" ++ isMember ++ fr") as mem", - fr"(" ++ memberCount ++ fr") as cnt" - ) - - val from = RFolder.table ++ fr"s INNER JOIN" ++ - Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(sOwner) - - val where = - sColl.is(account.collective) :: idQ.toList - .map(id => sId.is(id)) ::: nameQ.toList.map(q => - sName.lowerLike(s"%${q.toLowerCase}%") - ) ::: ownerLogin.toList.map(login => uLogin.is(login)) - - (cte ++ selectSimple(commas(cols), from, and(where) ++ orderBy(sName.asc))) + sql.run .query[FolderItem] .to[Vector] } /** Select all folder_id where the given account is member or owner. */ def findMemberFolderIds(account: AccountId): Fragment = { - val user = RUser.as("u") - val fId = RFolder.Columns.id.prefix("f") - val fOwner = RFolder.Columns.owner.prefix("f") - val fColl = RFolder.Columns.collective.prefix("f") - val uId = user.uid.column - val uLogin = user.login.column - val mFolder = RFolderMember.Columns.folder.prefix("m") - val mUser = RFolderMember.Columns.user.prefix("m") - - selectSimple( - Seq(fId), - RFolder.table ++ fr"f INNER JOIN" ++ Fragment.const( - user.tableName - ) ++ fr"u ON" ++ fOwner.is(uId), - and(fColl.is(account.collective), uLogin.is(account.user)) - ) ++ - fr"UNION ALL" ++ - selectSimple( - Seq(mFolder), - RFolderMember.table ++ fr"m INNER JOIN" ++ RFolder.table ++ fr"f ON" ++ fId.is( - mFolder - ) ++ - fr"INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ uId.is(mUser), - and(fColl.is(account.collective), uLogin.is(account.user)) + val user = RUser.as("u") + val f = RFolder.as("f") + val m = RFolderMember.as("m") + union( + Select( + select(f.id), + from(f).innerJoin(user, f.owner === user.uid), + f.collective === account.collective && user.login === account.user + ), + Select( + select(m.folder), + from(m) + .innerJoin(f, f.id === m.folder) + .innerJoin(user, user.uid === m.user), + f.collective === account.collective && user.login === account.user ) + ).run } def getMemberFolders(account: AccountId): ConnectionIO[Set[Ident]] = diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index b41ebfa6..b7c50f44 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -91,6 +91,7 @@ object QItem { val org = ROrganization.as("o") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") + val f = RFolder.as("f") val IC = RItem.Columns.all.map(_.prefix("i")) val OC = org.all.map(_.column) @@ -98,7 +99,7 @@ object QItem { val P1C = pers1.all.map(_.column) val EC = equip.all.map(_.oldColumn).map(_.prefix("e")) val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) - val FC = List(RFolder.Columns.id, RFolder.Columns.name).map(_.prefix("f")) + val FC = List(f.id.column, f.name.column) val cq = selectSimple( @@ -129,9 +130,11 @@ object QItem { fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo .prefix("i") .is(RItem.Columns.id.prefix("ref")) ++ - fr"LEFT JOIN" ++ RFolder.table ++ fr"f ON" ++ RItem.Columns.folder + fr"LEFT JOIN" ++ Fragment.const( + RFolder.T.tableName + ) ++ fr"f ON" ++ RItem.Columns.folder .prefix("i") - .is(RFolder.Columns.id.prefix("f")) ++ + .is(f.id.column) ++ fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) val q = cq @@ -322,13 +325,13 @@ object QItem { val org = ROrganization.as("o0") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") + val f = RFolder.as("f1") val IC = RItem.Columns val AC = RAttachment.Columns - val FC = RFolder.Columns val itemCols = IC.all val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) - val folderCols = List(FC.id, FC.name) + val folderCols = List(f.id.oldColumn, f.name.oldColumn) val cvItem = RCustomFieldValue.Columns.itemId.prefix("cv") val finalCols = commas( @@ -350,8 +353,8 @@ object QItem { pers1.name.column.f, equip.eid.oldColumn.prefix("e1").f, equip.name.oldColumn.prefix("e1").f, - FC.id.prefix("f1").f, - FC.name.prefix("f1").f, + f.id.column.f, + f.name.column.f, // sql uses 1 for first character IC.notes.prefix("i").substring(1, noteMaxLen), // last column is only for sorting @@ -384,7 +387,11 @@ object QItem { equip.cid.oldColumn.is(q.account.collective) ) val withFolder = - selectSimple(folderCols, RFolder.table, FC.collective.is(q.account.collective)) + selectSimple( + folderCols, + Fragment.const(f.tableName), + f.collective.oldColumn.is(q.account.collective) + ) val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" @@ -410,7 +417,7 @@ object QItem { fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment .prefix("i") .is(equip.eid.oldColumn.prefix("e1")) ++ - fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1")) ++ + fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(f.id.column) ++ (if (q.customValues.isEmpty) Fragment.empty else fr"INNER JOIN customvalues cv ON" ++ cvItem.is(IC.id.prefix("i"))) @@ -425,6 +432,7 @@ object QItem { val org = ROrganization.as("o0") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") + val f = RFolder.as("f1") val IC = RItem.Columns // inclusive tags are AND-ed @@ -468,7 +476,7 @@ object QItem { org.oid.column.isOrDiscard(q.corrOrg), pers1.pid.column.isOrDiscard(q.concPerson), equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip), - RFolder.Columns.id.prefix("f1").isOrDiscard(q.folder), + f.id.column.isOrDiscard(q.folder), if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl diff --git a/modules/store/src/main/scala/docspell/store/records/RContact.scala b/modules/store/src/main/scala/docspell/store/records/RContact.scala index 7fd64778..40053665 100644 --- a/modules/store/src/main/scala/docspell/store/records/RContact.scala +++ b/modules/store/src/main/scala/docspell/store/records/RContact.scala @@ -27,7 +27,7 @@ object RContact { val personId = Column[Ident]("pid", this) val orgId = Column[Ident]("oid", this) val created = Column[Timestamp]("created", this) - val all = List(contactId, value, kind, personId, orgId, created) + val all = List[Column[_]](contactId, value, kind, personId, orgId, created) } private val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RFolder.scala b/modules/store/src/main/scala/docspell/store/records/RFolder.scala index 0b3b0ebb..d678fe9f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFolder.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFolder.scala @@ -4,8 +4,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -26,61 +26,58 @@ object RFolder { now <- Timestamp.current[F] } yield RFolder(nId, name, account.collective, account.user, now) - val table = fr"folder" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "folder" - object Columns { - - val id = Column("id") - val name = Column("name") - val collective = Column("cid") - val owner = Column("owner") - val created = Column("created") + val id = Column[Ident]("id", this) + val name = Column[String]("name", this) + val collective = Column[Ident]("cid", this) + val owner = Column[Ident]("owner", this) + val created = Column[Timestamp]("created", this) val all = List(id, name, collective, owner, created) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(value: RFolder): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(value: RFolder): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${value.id},${value.name},${value.collectiveId},${value.owner},${value.created}" ) - sql.update.run - } def update(v: RFolder): ConnectionIO[Int] = - updateRow( - table, - and(id.is(v.id), collective.is(v.collectiveId), owner.is(v.owner)), - name.setTo(v.name) - ).update.run + DML.update( + T, + T.id === v.id && T.collective === v.collectiveId && T.owner === v.owner, + DML.set(T.name.setTo(v.name)) + ) def existsByName(coll: Ident, folderName: String): ConnectionIO[Boolean] = - selectCount(id, table, and(collective.is(coll), name.is(folderName))) + run(select(count(T.id)), from(T), T.collective === coll && T.name === folderName) .query[Int] .unique .map(_ > 0) def findById(folderId: Ident): ConnectionIO[Option[RFolder]] = { - val sql = selectSimple(all, table, id.is(folderId)) + val sql = run(select(T.all), from(T), T.id === folderId) sql.query[RFolder].option } def findAll( coll: Ident, nameQ: Option[String], - order: Columns.type => Column + order: Table => Column[_] ): ConnectionIO[Vector[RFolder]] = { - val q = Seq(collective.is(coll)) ++ (nameQ match { - case Some(str) => Seq(name.lowerLike(s"%${str.toLowerCase}%")) - case None => Seq.empty - }) - val sql = selectSimple(all, table, and(q)) ++ orderBy(order(Columns).f) - sql.query[RFolder].to[Vector] + val nameFilter = nameQ.map(n => T.name.like(s"%${n.toLowerCase}%")) + val sql = Select(select(T.all), from(T), T.collective === coll &&? nameFilter) + .orderBy(order(T)) + sql.run.query[RFolder].to[Vector] } def delete(folderId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(folderId)).update.run + DML.delete(T, T.id === folderId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala b/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala index cb7b5f21..f16d880d 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala @@ -4,8 +4,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -25,37 +25,36 @@ object RFolderMember { now <- Timestamp.current[F] } yield RFolderMember(nId, folder, user, now) - val table = fr"folder_member" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "folder_member" - object Columns { - - val id = Column("id") - val folder = Column("folder_id") - val user = Column("user_id") - val created = Column("created") + val id = Column[Ident]("id", this) + val folder = Column[Ident]("folder_id", this) + val user = Column[Ident]("user_id", this) + val created = Column[Timestamp]("created", this) val all = List(id, folder, user, created) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(value: RFolderMember): ConnectionIO[Int] = { - val sql = insertRow( - table, - all, + def insert(value: RFolderMember): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${value.id},${value.folderId},${value.userId},${value.created}" ) - sql.update.run - } def findByUserId(userId: Ident, folderId: Ident): ConnectionIO[Option[RFolderMember]] = - selectSimple(all, table, and(folder.is(folderId), user.is(userId))) + run(select(T.all), from(T), T.folder === folderId && T.user === userId) .query[RFolderMember] .option def delete(userId: Ident, folderId: Ident): ConnectionIO[Int] = - deleteFrom(table, and(folder.is(folderId), user.is(userId))).update.run + DML.delete(T, T.folder === folderId && T.user === userId) def deleteAll(folderId: Ident): ConnectionIO[Int] = - deleteFrom(table, folder.is(folderId)).update.run + DML.delete(T, T.folder === folderId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RTag.scala b/modules/store/src/main/scala/docspell/store/records/RTag.scala index 3285fbc4..577e6e5d 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTag.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala @@ -27,7 +27,7 @@ object RTag { val name = Column[String]("name", this) val category = Column[String]("category", this) val created = Column[Timestamp]("created", this) - val all = List(tid, cid, name, category, created) + val all = List[Column[_]](tid, cid, name, category, created) } val T = Table(None) def as(alias: String): Table = diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala index a5b22f81..2d920352 100644 --- a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala @@ -22,7 +22,7 @@ object DoobieQueryTest extends SimpleTestSuite { ) val q = Select(proj, table, cond) - val frag = DoobieQuery(q) + val frag = SelectBuilder(q) assertEquals( frag.toString, """Fragment("SELECT c.id, c.name, c.owner_id, c.lecturer_id, c.lessons FROM course c INNER JOIN person o ON c.owner_id = o.id LEFT JOIN person l ON c.lecturer_id = l.id WHERE (LOWER(c.name) LIKE ? AND o.name = ? )")""" From 613696539fdbb8064f20efcbe7ceb8ba8d4c0152 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 01:36:12 +0100 Subject: [PATCH 13/38] Minor refactorings --- .../docspell/joex/analysis/RegexNerFile.scala | 2 +- .../main/scala/docspell/store/qb/Column.scala | 6 - .../scala/docspell/store/qb/Condition.scala | 8 +- .../main/scala/docspell/store/qb/DML.scala | 20 ++-- .../main/scala/docspell/store/qb/DSL.scala | 63 +++++++---- .../scala/docspell/store/qb/FromExpr.scala | 7 +- .../scala/docspell/store/qb/GroupBy.scala | 5 + .../main/scala/docspell/store/qb/Select.scala | 21 +++- .../store/qb/impl/SelectBuilder.scala | 2 +- .../docspell/store/queries/QCollective.scala | 6 +- .../docspell/store/queries/QCustomField.scala | 81 +++++++------ .../docspell/store/queries/QFolder.scala | 106 +++++++++--------- .../scala/docspell/store/queries/QItem.scala | 75 ++++++------- .../scala/docspell/store/queries/QJob.scala | 14 +-- .../scala/docspell/store/queries/QMails.scala | 2 +- .../store/queries/QOrganization.scala | 4 +- .../store/queries/QPeriodicTask.scala | 2 +- .../docspell/store/records/RContact.scala | 4 +- .../docspell/store/records/RCustomField.scala | 76 +++++++------ .../store/records/RCustomFieldValue.scala | 58 +++++----- .../docspell/store/records/REquipment.scala | 8 +- .../docspell/store/records/RFolder.scala | 5 +- .../store/records/RFolderMember.scala | 3 +- .../store/records/RFtsMigration.scala | 3 +- .../docspell/store/records/RInvitation.scala | 3 +- .../scala/docspell/store/records/RItem.scala | 55 ++++++++- .../scala/docspell/store/records/RJob.scala | 4 +- .../docspell/store/records/RJobGroupUse.scala | 3 +- .../docspell/store/records/RJobLog.scala | 6 +- .../scala/docspell/store/records/RNode.scala | 3 +- .../store/records/ROrganization.scala | 19 +++- .../store/records/RPeriodicTask.scala | 3 +- .../docspell/store/records/RPerson.scala | 6 +- .../docspell/store/records/RRememberMe.scala | 3 +- .../docspell/store/records/RSentMail.scala | 2 +- .../store/records/RSentMailItem.scala | 5 +- .../docspell/store/records/RSource.scala | 6 +- .../scala/docspell/store/records/RTag.scala | 8 +- .../docspell/store/records/RTagItem.scala | 14 +-- .../docspell/store/records/RTagSource.scala | 3 +- .../scala/docspell/store/records/RUser.scala | 16 ++- .../docspell/store/records/RUserEmail.scala | 6 +- .../docspell/store/records/RUserImap.scala | 6 +- .../docspell/store/qb/QueryBuilderTest.scala | 14 ++- ...ueryTest.scala => SelectBuilderTest.scala} | 4 +- .../store/qb/model/CourseRecord.scala | 3 +- .../store/qb/model/PersonRecord.scala | 3 +- 47 files changed, 450 insertions(+), 326 deletions(-) rename modules/store/src/test/scala/docspell/store/qb/impl/{DoobieQueryTest.scala => SelectBuilderTest.scala} (90%) diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index e5dcce3e..75c21673 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -141,7 +141,7 @@ object RegexNerFile { def latestUpdate(collective: Ident): ConnectionIO[Option[Timestamp]] = { def max_(col: Column[_], cidCol: Column[Ident]): Select = - Select(List(max(col).as("t")), from(col.table), cidCol === collective) + Select(max(col).as("t"), from(col.table), cidCol === collective) val sql = union( max_(ROrganization.T.updated, ROrganization.T.cid), diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala index 60d49abc..3e59a62c 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Column.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -3,12 +3,6 @@ package docspell.store.qb case class Column[A](name: String, table: TableDef) { def inTable(t: TableDef): Column[A] = copy(table = t) - - def s: SelectExpr = - SelectExpr.SelectColumn(this, None) - - def as(alias: String): SelectExpr = - SelectExpr.SelectColumn(this, Some(alias)) } object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 264889f7..c89b7178 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -4,13 +4,7 @@ import cats.data.NonEmptyList import doobie._ -sealed trait Condition { - def s: SelectExpr.SelectCondition = - SelectExpr.SelectCondition(this, None) - - def as(alias: String): SelectExpr.SelectCondition = - SelectExpr.SelectCondition(this, Some(alias)) -} +sealed trait Condition object Condition { diff --git a/modules/store/src/main/scala/docspell/store/qb/DML.scala b/modules/store/src/main/scala/docspell/store/qb/DML.scala index e8fd6dcf..48a7f051 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DML.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DML.scala @@ -1,5 +1,7 @@ package docspell.store.qb +import cats.data.{NonEmptyList => Nel} + import docspell.store.qb.impl._ import doobie._ @@ -15,44 +17,44 @@ object DML { fr"DELETE FROM" ++ FromExprBuilder.buildTable(table) ++ fr"WHERE" ++ ConditionBuilder .build(cond) - def insert(table: TableDef, cols: Seq[Column[_]], values: Fragment): ConnectionIO[Int] = + def insert(table: TableDef, cols: Nel[Column[_]], values: Fragment): ConnectionIO[Int] = insertFragment(table, cols, List(values)).update.run def insertMany( table: TableDef, - cols: Seq[Column[_]], + cols: Nel[Column[_]], values: Seq[Fragment] ): ConnectionIO[Int] = insertFragment(table, cols, values).update.run def insertFragment( table: TableDef, - cols: Seq[Column[_]], + cols: Nel[Column[_]], values: Seq[Fragment] ): Fragment = fr"INSERT INTO" ++ FromExprBuilder.buildTable(table) ++ sql"(" ++ cols .map(SelectExprBuilder.columnNoPrefix) - .reduce(_ ++ comma ++ _) ++ fr") VALUES" ++ + .reduceLeft(_ ++ comma ++ _) ++ fr") VALUES" ++ values.map(f => sql"(" ++ f ++ sql")").reduce(_ ++ comma ++ _) def update( table: TableDef, cond: Condition, - setter: Seq[Setter[_]] + setter: Nel[Setter[_]] ): ConnectionIO[Int] = updateFragment(table, Some(cond), setter).update.run def updateFragment( table: TableDef, cond: Option[Condition], - setter: Seq[Setter[_]] + setter: Nel[Setter[_]] ): Fragment = { val condFrag = cond.map(SelectBuilder.cond).getOrElse(Fragment.empty) fr"UPDATE" ++ FromExprBuilder.buildTable(table) ++ fr"SET" ++ setter .map(s => buildSetter(s)) - .reduce(_ ++ comma ++ _) ++ + .reduceLeft(_ ++ comma ++ _) ++ condFrag } @@ -74,6 +76,6 @@ object DML { colFrag ++ fr" =" ++ colFrag ++ fr" + $amount" } - def set(s: Setter[_], more: Setter[_]*): Seq[Setter[_]] = - more :+ s + def set(s: Setter[_], more: Setter[_]*): Nel[Setter[_]] = + Nel(s, more.toList) } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 6223ebc0..7e388075 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -1,6 +1,6 @@ package docspell.store.qb -import cats.data.NonEmptyList +import cats.data.{NonEmptyList => Nel} import docspell.store.impl.DoobieMeta import docspell.store.qb.impl.SelectBuilder @@ -9,14 +9,14 @@ import doobie.{Fragment, Put} trait DSL extends DoobieMeta { - def run(projection: Seq[SelectExpr], from: FromExpr): Fragment = + def run(projection: Nel[SelectExpr], from: FromExpr): Fragment = SelectBuilder(Select(projection, from)) - def run(projection: Seq[SelectExpr], from: FromExpr, where: Condition): Fragment = + def run(projection: Nel[SelectExpr], from: FromExpr, where: Condition): Fragment = SelectBuilder(Select(projection, from, where)) def runDistinct( - projection: Seq[SelectExpr], + projection: Nel[SelectExpr], from: FromExpr, where: Condition ): Fragment = @@ -25,24 +25,30 @@ trait DSL extends DoobieMeta { def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl = DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector) - def select(cond: Condition): Seq[SelectExpr] = - Seq(SelectExpr.SelectCondition(cond, None)) + def select(cond: Condition): Nel[SelectExpr] = + Nel.of(SelectExpr.SelectCondition(cond, None)) - def select(dbf: DBFunction): Seq[SelectExpr] = - Seq(SelectExpr.SelectFun(dbf, None)) + def select(dbf: DBFunction): Nel[SelectExpr] = + Nel.of(SelectExpr.SelectFun(dbf, None)) - def select(e: SelectExpr, es: SelectExpr*): Seq[SelectExpr] = - es.prepended(e) + def select(e: SelectExpr, es: SelectExpr*): Nel[SelectExpr] = + Nel(e, es.toList) - def select(c: Column[_], cs: Column[_]*): Seq[SelectExpr] = - cs.prepended(c).map(col => SelectExpr.SelectColumn(col, None)) + def select(c: Column[_], cs: Column[_]*): Nel[SelectExpr] = + Nel(c, cs.toList).map(col => SelectExpr.SelectColumn(col, None)) - def select(seq: Seq[Column[_]], seqs: Seq[Column[_]]*): Seq[SelectExpr] = - (seq ++ seqs.flatten).map(c => SelectExpr.SelectColumn(c, None)) + def select(seq: Nel[Column[_]], seqs: Nel[Column[_]]*): Nel[SelectExpr] = + seqs.foldLeft(seq)(_ concatNel _).map(c => SelectExpr.SelectColumn(c, None)) def union(s1: Select, sn: Select*): Select = Select.Union(s1, sn.toVector) + def intersect(s1: Select, sn: Select*): Select = + Select.Intersect(s1, sn.toVector) + + def intersect(nel: Nel[Select]): Select = + Select.Intersect(nel.head, nel.tail.toVector) + def from(table: TableDef): FromExpr.From = FromExpr.From(table) @@ -105,8 +111,12 @@ trait DSL extends DoobieMeta { else and(c, cs: _*) implicit final class ColumnOps[A](col: Column[A]) { - def s: SelectExpr = SelectExpr.SelectColumn(col, None) - def as(alias: String) = SelectExpr.SelectColumn(col, Some(alias)) + def s: SelectExpr = + SelectExpr.SelectColumn(col, None) + def as(alias: String): SelectExpr = + SelectExpr.SelectColumn(col, Some(alias)) + def as(otherCol: Column[A]): SelectExpr = + SelectExpr.SelectColumn(col, Some(otherCol.name)) def setTo(value: A)(implicit P: Put[A]): Setter[A] = Setter.SetValue(col, value) @@ -153,20 +163,28 @@ trait DSL extends DoobieMeta { def in(subsel: Select): Condition = Condition.InSubSelect(col, subsel) - def in(values: NonEmptyList[A])(implicit P: Put[A]): Condition = + def in(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, false) - def inLower(values: NonEmptyList[A])(implicit P: Put[A]): Condition = + def inLower(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, true) def isNull: Condition = Condition.IsNull(col) + def isNotNull: Condition = + Condition.IsNull(col).negate + def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) } implicit final class ConditionOps(c: Condition) { + def s: SelectExpr = + SelectExpr.SelectCondition(c, None) + + def as(alias: String): SelectExpr = + SelectExpr.SelectCondition(c, Some(alias)) def &&(other: Condition): Condition = and(c, other) @@ -188,8 +206,10 @@ trait DSL extends DoobieMeta { } implicit final class DBFunctionOps(dbf: DBFunction) { - def s: SelectExpr = SelectExpr.SelectFun(dbf, None) - def as(alias: String) = SelectExpr.SelectFun(dbf, Some(alias)) + def s: SelectExpr = + SelectExpr.SelectFun(dbf, None) + def as(alias: String): SelectExpr = + SelectExpr.SelectFun(dbf, Some(alias)) def ===[A](value: A)(implicit P: Put[A]): Condition = Condition.CompareFVal(dbf, Operator.Eq, value) @@ -233,6 +253,9 @@ object DSL extends DSL { def select(s: Select): Select.WithCte = Select.WithCte(cte, ctes, s) + + def apply(s: Select): Select.WithCte = + select(s) } } diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index d78f3143..8a8d65b1 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -1,11 +1,6 @@ package docspell.store.qb -sealed trait FromExpr { - -// def innerJoin(other: TableDef, on: Condition): FromExpr -// -// def leftJoin(other: TableDef, on: Condition): FromExpr -} +sealed trait FromExpr object FromExpr { diff --git a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala index fffa53f9..8a5b451b 100644 --- a/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala +++ b/modules/store/src/main/scala/docspell/store/qb/GroupBy.scala @@ -1,5 +1,7 @@ package docspell.store.qb +import cats.data.NonEmptyList + case class GroupBy(name: SelectExpr, names: Vector[SelectExpr], having: Option[Condition]) object GroupBy { @@ -10,4 +12,7 @@ object GroupBy { cs.toVector.map(c => SelectExpr.SelectColumn(c, None)), None ) + + def apply(nel: NonEmptyList[Column[_]]): GroupBy = + apply(nel.head, nel.tail: _*) } diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index ac6d0ca1..406135f0 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -1,11 +1,13 @@ package docspell.store.qb +import cats.data.{NonEmptyList => Nel} + import docspell.store.qb.impl.SelectBuilder import doobie._ sealed trait Select { - def run: Fragment = + def build: Fragment = SelectBuilder(this) def as(alias: String): SelectExpr.SelectQuery = @@ -25,17 +27,26 @@ sealed trait Select { } object Select { - def apply(projection: Seq[SelectExpr], from: FromExpr) = + def apply(projection: Nel[SelectExpr], from: FromExpr) = SimpleSelect(false, projection, from, None, None) + def apply(projection: SelectExpr, from: FromExpr) = + SimpleSelect(false, Nel.of(projection), from, None, None) + def apply( - projection: Seq[SelectExpr], + projection: Nel[SelectExpr], from: FromExpr, where: Condition ) = SimpleSelect(false, projection, from, Some(where), None) def apply( - projection: Seq[SelectExpr], + projection: SelectExpr, + from: FromExpr, + where: Condition + ) = SimpleSelect(false, Nel.of(projection), from, Some(where), None) + + def apply( + projection: Nel[SelectExpr], from: FromExpr, where: Condition, groupBy: GroupBy @@ -43,7 +54,7 @@ object Select { case class SimpleSelect( distinctFlag: Boolean, - projection: Seq[SelectExpr], + projection: Nel[SelectExpr], from: FromExpr, where: Option[Condition], groupBy: Option[GroupBy] diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index 3d64c67b..23ca286e 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -40,7 +40,7 @@ object SelectBuilder { } def buildSimple(sq: Select.SimpleSelect): Fragment = { - val f0 = sq.projection.map(selectExpr).reduce(_ ++ comma ++ _) + val f0 = sq.projection.map(selectExpr).reduceLeft(_ ++ comma ++ _) val f1 = fromExpr(sq.from) val f2 = sq.where.map(cond).getOrElse(Fragment.empty) val f3 = sq.groupBy.map(groupBy).getOrElse(Fragment.empty) diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index 62b23ab0..36361780 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -84,12 +84,12 @@ object QCollective { val t = RTag.as("t") val sql = Select( - select(t.all) ++ select(count(ti.itemId)), + select(t.all).append(count(ti.itemId).s), from(ti).innerJoin(t, ti.tagId === t.tid), t.cid === coll ).group(GroupBy(t.name, t.tid, t.category)) - sql.run.query[TagCount].to[List] + sql.build.query[TagCount].to[List] } def getContacts( @@ -113,6 +113,6 @@ object QCollective { select(rc.all), from(rc), (rc.orgId.in(orgCond) || rc.personId.in(persCond)) &&? valueFilter &&? kindFilter - ).orderBy(rc.value).run.query[RContact].stream + ).orderBy(rc.value).build.query[RContact].stream } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala index 7b10964e..0990a12b 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala @@ -1,17 +1,17 @@ package docspell.store.queries -import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ -import doobie.implicits._ object QCustomField { + private val f = RCustomField.as("f") + private val v = RCustomFieldValue.as("v") case class CustomFieldData(field: RCustomField, usageCount: Int) @@ -19,46 +19,57 @@ object QCustomField { coll: Ident, nameQuery: Option[String] ): ConnectionIO[Vector[CustomFieldData]] = - findFragment(coll, nameQuery, None).query[CustomFieldData].to[Vector] + findFragment(coll, nameQuery, None).build.query[CustomFieldData].to[Vector] def findById(field: Ident, collective: Ident): ConnectionIO[Option[CustomFieldData]] = - findFragment(collective, None, field.some).query[CustomFieldData].option + findFragment(collective, None, field.some).build.query[CustomFieldData].option private def findFragment( coll: Ident, nameQuery: Option[String], fieldId: Option[Ident] - ): Fragment = { - val fId = RCustomField.Columns.id.prefix("f") - val fColl = RCustomField.Columns.cid.prefix("f") - val fName = RCustomField.Columns.name.prefix("f") - val fLabel = RCustomField.Columns.label.prefix("f") - val vField = RCustomFieldValue.Columns.field.prefix("v") + ): Select = { +// val fId = RCustomField.Columns.id.prefix("f") +// val fColl = RCustomField.Columns.cid.prefix("f") +// val fName = RCustomField.Columns.name.prefix("f") +// val fLabel = RCustomField.Columns.label.prefix("f") +// val vField = RCustomFieldValue.Columns.field.prefix("v") +// +// val join = RCustomField.table ++ fr"f LEFT OUTER JOIN" ++ +// RCustomFieldValue.table ++ fr"v ON" ++ fId.is(vField) +// +// val cols = RCustomField.Columns.all.map(_.prefix("f")) :+ Column("COUNT(v.id)") +// +// val nameCond = nameQuery.map(QueryWildcard.apply) match { +// case Some(q) => +// or(fName.lowerLike(q), and(fLabel.isNotNull, fLabel.lowerLike(q))) +// case None => +// Fragment.empty +// } +// val fieldCond = fieldId match { +// case Some(id) => +// fId.is(id) +// case None => +// Fragment.empty +// } +// val cond = and(fColl.is(coll), nameCond, fieldCond) +// +// val group = NonEmptyList.fromList(RCustomField.Columns.all) match { +// case Some(nel) => groupBy(nel.map(_.prefix("f"))) +// case None => Fragment.empty +// } +// +// selectSimple(cols, join, cond) ++ group - val join = RCustomField.table ++ fr"f LEFT OUTER JOIN" ++ - RCustomFieldValue.table ++ fr"v ON" ++ fId.is(vField) - - val cols = RCustomField.Columns.all.map(_.prefix("f")) :+ Column("COUNT(v.id)") - - val nameCond = nameQuery.map(QueryWildcard.apply) match { - case Some(q) => - or(fName.lowerLike(q), and(fLabel.isNotNull, fLabel.lowerLike(q))) - case None => - Fragment.empty - } - val fieldCond = fieldId match { - case Some(id) => - fId.is(id) - case None => - Fragment.empty - } - val cond = and(fColl.is(coll), nameCond, fieldCond) - - val group = NonEmptyList.fromList(RCustomField.Columns.all) match { - case Some(nel) => groupBy(nel.map(_.prefix("f"))) - case None => Fragment.empty + val nameFilter = nameQuery.map { q => + f.name.likes(q) || (f.label.isNotNull && f.label.like(q)) } - selectSimple(cols, join, cond) ++ group + Select( + f.all.map(_.s).append(count(v.id).as("num")), + from(f).leftJoin(v, f.id === v.field), + f.cid === coll &&? nameFilter &&? fieldId.map(fid => f.id === fid), + GroupBy(f.all) + ) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala index 79b6341f..08192cf3 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QFolder.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QFolder.scala @@ -183,68 +183,62 @@ object QFolder { // inner join user_ u on u.uid = s.owner // where s.cid = 'eike'; - val user = RUser.as("u") - val member = RFolderMember.as("m") - val folder = RFolder.as("s") - val memlogin = TableDef("memberlogin") - val memloginFolder = member.folder.inTable(memlogin) - val memloginLogn = user.login.inTable(memlogin) + val user = RUser.as("u") + val member = RFolderMember.as("m") + val folder = RFolder.as("s") + val memlogin = TableDef("memberlogin") + val mlFolder = Column[Ident]("folder", memlogin) + val mlLogin = Column[Ident]("login", memlogin) - val sql = - withCte( - memlogin -> union( - Select( - select(member.folder, user.login), - from(member) - .innerJoin(user, user.uid === member.user) - .innerJoin(folder, folder.id === member.folder), - folder.collective === account.collective - ), - Select( - select(folder.id, user.login), - from(folder) - .innerJoin(user, user.uid === folder.owner), - folder.collective === account.collective - ) + withCte( + memlogin -> union( + Select( + select(member.folder.as(mlFolder), user.login.as(mlLogin)), + from(member) + .innerJoin(user, user.uid === member.user) + .innerJoin(folder, folder.id === member.folder), + folder.collective === account.collective + ), + Select( + select(folder.id.as(mlFolder), user.login.as(mlLogin)), + from(folder) + .innerJoin(user, user.uid === folder.owner), + folder.collective === account.collective ) ) - .select( + )( + Select( + select( + folder.id.s, + folder.name.s, + folder.owner.s, + user.login.s, + folder.created.s, Select( - select( - folder.id.s, - folder.name.s, - folder.owner.s, - user.login.s, - folder.created.s, - Select( - select(countAll > 0), - from(memlogin), - memloginFolder === folder.id && memloginLogn === account.user - ).as("member"), - Select( - select(countAll - 1), - from(memlogin), - memloginFolder === folder.id - ).as("member_count") - ), - from(folder) - .innerJoin(user, user.uid === folder.owner), - where( - folder.collective === account.collective &&? - idQ.map(id => folder.id === id) &&? - nameQ.map(q => folder.name.like(s"%${q.toLowerCase}%")) &&? - ownerLogin.map(login => user.login === login) - ) - ).orderBy(folder.name.asc) + select(countAll > 0), + from(memlogin), + mlFolder === folder.id && mlLogin === account.user + ).as("member"), + Select( + select(countAll - 1), + from(memlogin), + mlFolder === folder.id + ).as("member_count") + ), + from(folder) + .innerJoin(user, user.uid === folder.owner), + where( + folder.collective === account.collective &&? + idQ.map(id => folder.id === id) &&? + nameQ.map(q => folder.name.like(s"%${q.toLowerCase}%")) &&? + ownerLogin.map(login => user.login === login) ) - - sql.run - .query[FolderItem] - .to[Vector] + ).orderBy(folder.name.asc) + ).build.query[FolderItem].to[Vector] } /** Select all folder_id where the given account is member or owner. */ - def findMemberFolderIds(account: AccountId): Fragment = { + def findMemberFolderIds(account: AccountId): Select = { val user = RUser.as("u") val f = RFolder.as("f") val m = RFolderMember.as("m") @@ -261,11 +255,11 @@ object QFolder { .innerJoin(user, user.uid === m.user), f.collective === account.collective && user.login === account.user ) - ).run + ) } def getMemberFolders(account: AccountId): ConnectionIO[Set[Ident]] = - findMemberFolderIds(account).query[Ident].to[Set] + findMemberFolderIds(account).build.query[Ident].to[Set] private def findUserId(account: AccountId): ConnectionIO[Option[Ident]] = RUser.findByAccount(account).map(_.map(_.uid)) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index b7c50f44..da2981e9 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -12,6 +12,7 @@ import docspell.common.{IdRef, _} import docspell.store.Store import docspell.store.impl.Implicits._ import docspell.store.impl._ +import docspell.store.qb.Select import docspell.store.records._ import bitpeace.FileMeta @@ -94,10 +95,10 @@ object QItem { val f = RFolder.as("f") val IC = RItem.Columns.all.map(_.prefix("i")) - val OC = org.all.map(_.column) - val P0C = pers0.all.map(_.column) - val P1C = pers1.all.map(_.column) - val EC = equip.all.map(_.oldColumn).map(_.prefix("e")) + val OC = org.all.map(_.column).toList + val P0C = pers0.all.map(_.column).toList + val P1C = pers1.all.map(_.column).toList + val EC = equip.all.map(_.oldColumn).map(_.prefix("e")).toList val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) val FC = List(f.id.column, f.name.column) @@ -172,23 +173,17 @@ object QItem { def findCustomFieldValuesForItem( itemId: Ident ): ConnectionIO[Vector[ItemFieldValue]] = { - val cfId = RCustomField.Columns.id.prefix("cf") - val cfName = RCustomField.Columns.name.prefix("cf") - val cfLabel = RCustomField.Columns.label.prefix("cf") - val cfType = RCustomField.Columns.ftype.prefix("cf") - val cvItem = RCustomFieldValue.Columns.itemId.prefix("cvf") - val cvValue = RCustomFieldValue.Columns.value.prefix("cvf") - val cvField = RCustomFieldValue.Columns.field.prefix("cvf") + import docspell.store.qb.DSL._ - val cfFrom = - RCustomFieldValue.table ++ fr"cvf INNER JOIN" ++ RCustomField.table ++ fr"cf ON" ++ cvField - .is(cfId) + val cf = RCustomField.as("cf") + val cv = RCustomFieldValue.as("cvf") - selectSimple( - Seq(cfId, cfName, cfLabel, cfType, cvValue), - cfFrom, - cvItem.is(itemId) - ).query[ItemFieldValue].to[Vector] + Select( + select(cf.id, cf.name, cf.label, cf.ftype, cv.value), + from(cv) + .innerJoin(cf, cf.id === cv.field), + cv.itemId === itemId + ).build.query[ItemFieldValue].to[Vector] } case class ListItem( @@ -287,31 +282,30 @@ object QItem { private def findCustomFieldValuesForColl( coll: Ident, - cv: Seq[CustomValue] + values: Seq[CustomValue] ): Seq[(String, Fragment)] = { - val cfId = RCustomField.Columns.id.prefix("cf") - val cfName = RCustomField.Columns.name.prefix("cf") - val cfColl = RCustomField.Columns.cid.prefix("cf") - val cvValue = RCustomFieldValue.Columns.value.prefix("cvf") - val cvField = RCustomFieldValue.Columns.field.prefix("cvf") - val cvItem = RCustomFieldValue.Columns.itemId.prefix("cvf") + import docspell.store.qb.DSL._ - val cfFrom = - RCustomFieldValue.table ++ fr"cvf INNER JOIN" ++ RCustomField.table ++ fr"cf ON" ++ cvField - .is(cfId) + val cf = RCustomField.as("cf") + val cv = RCustomFieldValue.as("cv") def singleSelect(v: CustomValue) = - selectSimple( - Seq(cvItem), - cfFrom, - and( - cfColl.is(coll), - or(cfName.is(v.field), cfId.is(v.field)), - cvValue.lowerLike(QueryWildcard(v.value.toLowerCase)) + Select( + cv.itemId.s, + from(cv).innerJoin(cf, cv.field === cf.id), + where( + cf.cid === coll && + (cf.name === v.field || cf.id === v.field) && + cv.value.like(QueryWildcard(v.value.toLowerCase)) ) ) - if (cv.isEmpty) Seq.empty - else Seq("customvalues" -> cv.map(singleSelect).reduce(_ ++ fr"INTERSECT" ++ _)) + + NonEmptyList.fromList(values.toList) match { + case Some(nel) => + Seq("customvalues" -> intersect(nel.map(singleSelect)).build) + case None => + Seq.empty + } } private def findItemsBase( @@ -326,13 +320,14 @@ object QItem { val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") val f = RFolder.as("f1") + val cv = RCustomFieldValue.as("cv") val IC = RItem.Columns val AC = RAttachment.Columns val itemCols = IC.all val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) val folderCols = List(f.id.oldColumn, f.name.oldColumn) - val cvItem = RCustomFieldValue.Columns.itemId.prefix("cv") + val cvItem = cv.itemId.column val finalCols = commas( Seq( @@ -504,7 +499,7 @@ object QItem { .getOrElse(IC.id.prefix("i").is("")) ) .getOrElse(Fragment.empty), - or(iFolder.isNull, iFolder.isIn(QFolder.findMemberFolderIds(q.account))) + or(iFolder.isNull, iFolder.isIn(QFolder.findMemberFolderIds(q.account).build)) ) val order = q.orderAsc match { diff --git a/modules/store/src/main/scala/docspell/store/queries/QJob.scala b/modules/store/src/main/scala/docspell/store/queries/QJob.scala index 815f9de6..c85080a1 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QJob.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QJob.scala @@ -100,20 +100,20 @@ object QJob { val sql1 = Select( - List(max(JC.group).as("g")), + max(JC.group).as("g"), from(JC).innerJoin(G, JC.group === G.group), G.worker === worker && stateCond ) val sql2 = - Select(List(min(JC.group).as("g")), from(JC), stateCond) + Select(min(JC.group).as("g"), from(JC), stateCond) val gcol = Column[String]("g", TableDef("")) val groups = Select(select(gcol), fromSubSelect(union(sql1, sql2)).as("t0"), gcol.isNull.negate) // either 0, one or two results, but may be empty if RJob table is empty - groups.run.query[Ident].to[List].map(_.headOption) + groups.build.query[Ident].to[List].map(_.headOption) } private def stuckTriggerValue(t: RJob.Table, initialPause: Duration, now: Timestamp) = @@ -144,7 +144,7 @@ object QJob { (JC.state === stuck && stuckTrigger < now.toMillis)) ).orderBy(JC.state.asc, psort, JC.submitted.asc).limit(1) - sql.run.query[RJob].option + sql.build.query[RJob].option } def setCancelled[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = @@ -215,13 +215,13 @@ object QJob { select(JC.all), from(JC), JC.group === collective && JC.state.in(running) - ).orderBy(JC.submitted.desc).run.query[RJob].stream + ).orderBy(JC.submitted.desc).build.query[RJob].stream val waitingJobs = Select( select(JC.all), from(JC), JC.group === collective && JC.state.in(waiting) && JC.submitted > refDate - ).orderBy(JC.submitted.desc).run.query[RJob].stream.take(max) + ).orderBy(JC.submitted.desc).build.query[RJob].stream.take(max) val doneJobs = Select( select(JC.all), @@ -231,7 +231,7 @@ object QJob { JC.state.in(JobState.done), JC.submitted > refDate ) - ).orderBy(JC.submitted.desc).run.query[RJob].stream.take(max) + ).orderBy(JC.submitted.desc).build.query[RJob].stream.take(max) runningJobs ++ waitingJobs ++ doneJobs } diff --git a/modules/store/src/main/scala/docspell/store/queries/QMails.scala b/modules/store/src/main/scala/docspell/store/queries/QMails.scala index 06b528fa..30d476af 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QMails.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QMails.scala @@ -65,7 +65,7 @@ object QMails { mUser ) - (cols, from) + (cols.toList, from) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala index 6d41a841..585d2fd0 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala @@ -32,7 +32,7 @@ object QOrganization { org.cid === coll &&? valFilter ).orderBy(order(org)) - sql.run + sql.build .query[(ROrganization, Option[RContact])] .stream .groupAdjacentBy(_._1) @@ -86,7 +86,7 @@ object QOrganization { pers.cid === coll &&? valFilter ).orderBy(order(pers)) - sql.run + sql.build .query[(RPerson, Option[ROrganization], Option[RContact])] .stream .groupAdjacentBy(_._1) diff --git a/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala index 46d8a273..9cccf026 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala @@ -49,7 +49,7 @@ object QPeriodicTask { case None => RT.enabled === true } val sql = - Select(select(RT.all), from(RT), where).orderBy(RT.nextrun.asc).run + Select(select(RT.all), from(RT), where).orderBy(RT.nextrun.asc).build sql.query[RPeriodicTask].streamWithChunkSize(2).take(1).compile.last } diff --git a/modules/store/src/main/scala/docspell/store/records/RContact.scala b/modules/store/src/main/scala/docspell/store/records/RContact.scala index 40053665..e625a86c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RContact.scala +++ b/modules/store/src/main/scala/docspell/store/records/RContact.scala @@ -1,5 +1,7 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -27,7 +29,7 @@ object RContact { val personId = Column[Ident]("pid", this) val orgId = Column[Ident]("oid", this) val created = Column[Timestamp]("created", this) - val all = List[Column[_]](contactId, value, kind, personId, orgId, created) + val all = NonEmptyList.of[Column[_]](contactId, value, kind, personId, orgId, created) } private val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RCustomField.scala b/modules/store/src/main/scala/docspell/store/records/RCustomField.scala index f74c7cc3..e63c6b9c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RCustomField.scala +++ b/modules/store/src/main/scala/docspell/store/records/RCustomField.scala @@ -1,10 +1,11 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -19,58 +20,63 @@ case class RCustomField( ) object RCustomField { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "custom_field" - val table = fr"custom_field" + val id = Column[Ident]("id", this) + val name = Column[Ident]("name", this) + val label = Column[String]("label", this) + val cid = Column[Ident]("cid", this) + val ftype = Column[CustomFieldType]("ftype", this) + val created = Column[Timestamp]("created", this) - object Columns { - - val id = Column("id") - val name = Column("name") - val label = Column("label") - val cid = Column("cid") - val ftype = Column("ftype") - val created = Column("created") - - val all = List(id, name, label, cid, ftype, created) + val all = NonEmptyList.of[Column[_]](id, name, label, cid, ftype, created) } - import Columns._ - def insert(value: RCustomField): ConnectionIO[Int] = { - val sql = insertRow( - table, - Columns.all, + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + + def insert(value: RCustomField): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${value.id},${value.name},${value.label},${value.cid},${value.ftype},${value.created}" ) - sql.update.run - } def exists(fname: Ident, coll: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(name.is(fname), cid.is(coll))).query[Int].unique.map(_ > 0) + run(select(count(T.id)), from(T), T.name === fname && T.cid === coll) + .query[Int] + .unique + .map(_ > 0) def findById(fid: Ident, coll: Ident): ConnectionIO[Option[RCustomField]] = - selectSimple(all, table, and(id.is(fid), cid.is(coll))).query[RCustomField].option + run(select(T.all), from(T), T.id === fid && T.cid === coll).query[RCustomField].option def findByIdOrName(idOrName: Ident, coll: Ident): ConnectionIO[Option[RCustomField]] = - selectSimple(all, table, and(cid.is(coll), or(id.is(idOrName), name.is(idOrName)))) - .query[RCustomField] - .option + Select( + select(T.all), + from(T), + T.cid === coll && (T.id === idOrName || T.name === idOrName) + ).build.query[RCustomField].option def deleteById(fid: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(id.is(fid), cid.is(coll))).update.run + DML.delete(T, T.id === fid && T.cid === coll) def findAll(coll: Ident): ConnectionIO[Vector[RCustomField]] = - selectSimple(all, table, cid.is(coll)).query[RCustomField].to[Vector] + run(select(T.all), from(T), T.cid === coll).query[RCustomField].to[Vector] def update(value: RCustomField): ConnectionIO[Int] = - updateRow( - table, - and(id.is(value.id), cid.is(value.cid)), - commas( - name.setTo(value.name), - label.setTo(value.label), - ftype.setTo(value.ftype) + DML + .update( + T, + T.id === value.id && T.cid === value.cid, + DML.set( + T.name.setTo(value.name), + T.label.setTo(value.label), + T.ftype.setTo(value.ftype) + ) ) - ).update.run def setValue(f: RCustomField, item: Ident, fval: String): ConnectionIO[Int] = for { diff --git a/modules/store/src/main/scala/docspell/store/records/RCustomFieldValue.scala b/modules/store/src/main/scala/docspell/store/records/RCustomFieldValue.scala index 8830dc58..3a5eaa2c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RCustomFieldValue.scala +++ b/modules/store/src/main/scala/docspell/store/records/RCustomFieldValue.scala @@ -3,8 +3,8 @@ package docspell.store.records import cats.data.NonEmptyList import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -17,51 +17,51 @@ case class RCustomFieldValue( ) object RCustomFieldValue { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "custom_field_value" - val table = fr"custom_field_value" + val id = Column[Ident]("id", this) + val itemId = Column[Ident]("item_id", this) + val field = Column[Ident]("field", this) + val value = Column[String]("field_value", this) - object Columns { - - val id = Column("id") - val itemId = Column("item_id") - val field = Column("field") - val value = Column("field_value") - - val all = List(id, itemId, field, value) + val all = NonEmptyList.of[Column[_]](id, itemId, field, value) } - def insert(value: RCustomFieldValue): ConnectionIO[Int] = { - val sql = insertRow( - table, - Columns.all, + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + + def insert(value: RCustomFieldValue): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${value.id},${value.itemId},${value.field},${value.value}" ) - sql.update.run - } def updateValue( fieldId: Ident, item: Ident, value: String ): ConnectionIO[Int] = - updateRow( - table, - and(Columns.itemId.is(item), Columns.field.is(fieldId)), - Columns.value.setTo(value) - ).update.run + DML.update( + T, + T.itemId === item && T.field === fieldId, + DML.set(T.value.setTo(value)) + ) def countField(fieldId: Ident): ConnectionIO[Int] = - selectCount(Columns.id, table, Columns.field.is(fieldId)).query[Int].unique + Select(count(T.id).s, from(T), T.field === fieldId).build.query[Int].unique def deleteByField(fieldId: Ident): ConnectionIO[Int] = - deleteFrom(table, Columns.field.is(fieldId)).update.run + DML.delete(T, T.field === fieldId) def deleteByItem(item: Ident): ConnectionIO[Int] = - deleteFrom(table, Columns.itemId.is(item)).update.run + DML.delete(T, T.itemId === item) def deleteValue(fieldId: Ident, items: NonEmptyList[Ident]): ConnectionIO[Int] = - deleteFrom( - table, - and(Columns.field.is(fieldId), Columns.itemId.isIn(items)) - ).update.run + DML.delete( + T, + T.field === fieldId && T.itemId.in(items) + ) } diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala index 3e0276ea..d1cfa83a 100644 --- a/modules/store/src/main/scala/docspell/store/records/REquipment.scala +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -1,5 +1,7 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -24,7 +26,7 @@ object REquipment { val name = Column[String]("name", this) val created = Column[Timestamp]("created", this) val updated = Column[Timestamp]("updated", this) - val all = List(eid, cid, name, created, updated) + val all = NonEmptyList.of[Column[_]](eid, cid, name, created, updated) } val T = Table(None) @@ -81,13 +83,13 @@ object REquipment { .map(str => s"%${str.toLowerCase}%") .map(v => t.name.like(v)) - val sql = Select(select(t.all), from(t), q).orderBy(order(t)).run + val sql = Select(select(t.all), from(t), q).orderBy(order(t)).build sql.query[REquipment].to[Vector] } def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = { val t = Table(None) - run(select(List(t.eid, t.name)), from(t), t.cid === coll && t.name.like(equipName)) + run(select(t.eid, t.name), from(t), t.cid === coll && t.name.like(equipName)) .query[IdRef] .to[Vector] } diff --git a/modules/store/src/main/scala/docspell/store/records/RFolder.scala b/modules/store/src/main/scala/docspell/store/records/RFolder.scala index d678fe9f..a83ef1a6 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFolder.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFolder.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -35,7 +36,7 @@ object RFolder { val owner = Column[Ident]("owner", this) val created = Column[Timestamp]("created", this) - val all = List(id, name, collective, owner, created) + val all = NonEmptyList.of[Column[_]](id, name, collective, owner, created) } val T = Table(None) @@ -75,7 +76,7 @@ object RFolder { val nameFilter = nameQ.map(n => T.name.like(s"%${n.toLowerCase}%")) val sql = Select(select(T.all), from(T), T.collective === coll &&? nameFilter) .orderBy(order(T)) - sql.run.query[RFolder].to[Vector] + sql.build.query[RFolder].to[Vector] } def delete(folderId: Ident): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala b/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala index f16d880d..2e38ab74 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFolderMember.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -33,7 +34,7 @@ object RFolderMember { val user = Column[Ident]("user_id", this) val created = Column[Timestamp]("created", this) - val all = List(id, folder, user, created) + val all = NonEmptyList.of[Column[_]](id, folder, user, created) } val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala b/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala index b2f21930..18183b1b 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFtsMigration.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -39,7 +40,7 @@ object RFtsMigration { val description = Column[String]("description", this) val created = Column[Timestamp]("created", this) - val all = List(id, version, ftsEngine, description, created) + val all = NonEmptyList.of[Column[_]](id, version, ftsEngine, description, created) } val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala index f3243566..bacecc0f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala +++ b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ @@ -18,7 +19,7 @@ object RInvitation { val id = Column[Ident]("id", this) val created = Column[Timestamp]("created", this) - val all = List(id, created) + val all = NonEmptyList.of[Column[_]](id, created) } val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 980cd324..eba4db3a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -7,6 +7,7 @@ import cats.implicits._ import docspell.common._ import docspell.store.impl.Implicits._ import docspell.store.impl._ +import docspell.store.qb.{Select, TableDef} import doobie._ import doobie.implicits._ @@ -63,6 +64,51 @@ object RItem { None ) + final case class Table(alias: Option[String]) extends TableDef { + import docspell.store.qb.Column + val tableName = "item" + + val id = Column[Ident]("itemid", this) + val cid = Column[Ident]("cid", this) + val name = Column[String]("name", this) + val itemDate = Column[Timestamp]("itemdate", this) + val source = Column[String]("source", this) + val incoming = Column[Direction]("incoming", this) + val state = Column[ItemState]("state", this) + val corrOrg = Column[Ident]("corrorg", this) + val corrPerson = Column[Ident]("corrperson", this) + val concPerson = Column[Ident]("concperson", this) + val concEquipment = Column[Ident]("concequipment", this) + val inReplyTo = Column[Ident]("inreplyto", this) + val dueDate = Column[Timestamp]("duedate", this) + val created = Column[Timestamp]("created", this) + val updated = Column[Timestamp]("updated", this) + val notes = Column[String]("notes", this) + val folder = Column[Ident]("folder_id", this) + val all = NonEmptyList.of[Column[_]]( + id, + cid, + name, + itemDate, + source, + incoming, + state, + corrOrg, + corrPerson, + concPerson, + concEquipment, + inReplyTo, + dueDate, + created, + updated, + notes, + folder + ) + } + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + val table = fr"item" object Columns { @@ -349,9 +395,12 @@ object RItem { updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run } - def filterItemsFragment(items: NonEmptyList[Ident], coll: Ident): Fragment = - selectSimple(Seq(id), table, and(cid.is(coll), id.isIn(items))) + def filterItemsFragment(items: NonEmptyList[Ident], coll: Ident): Select = { + import docspell.store.qb.DSL._ + + Select(select(T.id), from(T), T.cid === coll && T.id.in(items)) + } def filterItems(items: NonEmptyList[Ident], coll: Ident): ConnectionIO[Vector[Ident]] = - filterItemsFragment(items, coll).query[Ident].to[Vector] + filterItemsFragment(items, coll).build.query[Ident].to[Vector] } diff --git a/modules/store/src/main/scala/docspell/store/records/RJob.scala b/modules/store/src/main/scala/docspell/store/records/RJob.scala index 5f7b8850..e9112c5a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJob.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJob.scala @@ -90,7 +90,7 @@ object RJob { val started = Column[Timestamp]("started", this) val startedmillis = Column[Long]("startedmillis", this) val finished = Column[Timestamp]("finished", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, task, group, @@ -263,7 +263,7 @@ object RJob { def selectGroupInState(states: NonEmptyList[JobState]): ConnectionIO[Vector[Ident]] = { val sql = Select(select(T.group), from(T), T.state.in(states)).orderBy(T.group) - sql.run.query[Ident].to[Vector] + sql.build.query[Ident].to[Vector] } def delete(jobId: Ident): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala index 9cf4aec4..8763753c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ @@ -17,7 +18,7 @@ object RJobGroupUse { val group = Column[Ident]("groupid", this) val worker = Column[Ident]("workerid", this) - val all = List(group, worker) + val all = NonEmptyList.of[Column[_]](group, worker) } val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala index 999e9570..3e14bda7 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala @@ -1,5 +1,7 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -24,7 +26,7 @@ object RJobLog { val level = Column[LogLevel]("level", this) val created = Column[Timestamp]("created", this) val message = Column[String]("message", this) - val all = List(id, jobId, level, created, message) + val all = NonEmptyList.of[Column[_]](id, jobId, level, created, message) // separate column only for sorting, so not included in `all` and // the case class @@ -45,7 +47,7 @@ object RJobLog { def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] = Select(select(T.all), from(T), T.jobId === id) .orderBy(T.created.asc, T.counter.asc) - .run + .build .query[RJobLog] .to[Vector] diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index 8bc7bff1..4c3838fe 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ @@ -31,7 +32,7 @@ object RNode { val url = Column[LenientUri]("url", this) val updated = Column[Timestamp]("updated", this) val created = Column[Timestamp]("created", this) - val all = List(id, nodeType, url, updated, created) + val all = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created) } def as(alias: String): Table = diff --git a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala index 2aa0a743..0e8efafd 100644 --- a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala +++ b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala @@ -1,6 +1,7 @@ package docspell.store.records import cats.Eq +import cats.data.NonEmptyList import fs2.Stream import docspell.common.{IdRef, _} @@ -40,7 +41,19 @@ object ROrganization { val notes = Column[String]("notes", this) val created = Column[Timestamp]("created", this) val updated = Column[Timestamp]("updated", this) - val all = List(oid, cid, name, street, zip, city, country, notes, created, updated) + val all = + NonEmptyList.of[Column[_]]( + oid, + cid, + name, + street, + zip, + city, + country, + notes, + created, + updated + ) } val T = Table(None) @@ -120,7 +133,7 @@ object ROrganization { order: Table => Column[_] ): Stream[ConnectionIO, ROrganization] = { val sql = Select(select(T.all), from(T), T.cid === coll).orderBy(order(T)) - sql.run.query[ROrganization].stream + sql.build.query[ROrganization].stream } def findAllRef( @@ -131,7 +144,7 @@ object ROrganization { val nameFilter = nameQ.map(s => T.name.like(s"%${s.toLowerCase}%")) val sql = Select(select(T.oid, T.name), from(T), T.cid === coll &&? nameFilter) .orderBy(order(T)) - sql.run.query[IdRef].to[Vector] + sql.build.query[IdRef].to[Vector] } def delete(id: Ident, coll: Ident): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala index e0dcdb3f..8e0383cf 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -123,7 +124,7 @@ object RPeriodicTask { val timer = Column[CalEvent]("timer", this) val nextrun = Column[Timestamp]("nextrun", this) val created = Column[Timestamp]("created", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, enabled, task, diff --git a/modules/store/src/main/scala/docspell/store/records/RPerson.scala b/modules/store/src/main/scala/docspell/store/records/RPerson.scala index 04eb7831..cb2d2909 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPerson.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPerson.scala @@ -46,7 +46,7 @@ object RPerson { val created = Column[Timestamp]("created", this) val updated = Column[Timestamp]("updated", this) val oid = Column[Ident]("oid", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( pid, cid, name, @@ -150,7 +150,7 @@ object RPerson { order: Table => Column[_] ): Stream[ConnectionIO, RPerson] = { val sql = Select(select(T.all), from(T), T.cid === coll).orderBy(order(T)) - sql.run.query[RPerson].stream + sql.build.query[RPerson].stream } def findAllRef( @@ -163,7 +163,7 @@ object RPerson { val sql = Select(select(T.pid, T.name), from(T), T.cid === coll &&? nameFilter) .orderBy(order(T)) - sql.run.query[IdRef].to[Vector] + sql.build.query[IdRef].to[Vector] } def delete(personId: Ident, coll: Ident): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala b/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala index 464c0576..1e0606ff 100644 --- a/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala +++ b/modules/store/src/main/scala/docspell/store/records/RRememberMe.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ @@ -21,7 +22,7 @@ object RRememberMe { val username = Column[Ident]("login", this) val created = Column[Timestamp]("created", this) val uses = Column[Int]("uses", this) - val all = List(id, cid, username, created, uses) + val all = NonEmptyList.of[Column[_]](id, cid, username, created, uses) } private val T = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RSentMail.scala b/modules/store/src/main/scala/docspell/store/records/RSentMail.scala index cd4aa224..f2aa1d72 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSentMail.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSentMail.scala @@ -92,7 +92,7 @@ object RSentMail { val body = Column[String]("body", this) val created = Column[Timestamp]("created", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, uid, messageId, diff --git a/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala b/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala index 04dffc0b..d648567b 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSentMailItem.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -37,7 +38,7 @@ object RSentMailItem { val sentMailId = Column[Ident]("sentmail_id", this) val created = Column[Timestamp]("created", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, itemId, sentMailId, @@ -60,7 +61,7 @@ object RSentMailItem { DML.delete(T, T.sentMailId === mailId) def findSentMailIdsByItem(item: Ident): ConnectionIO[Set[Ident]] = - run(select(Seq(T.sentMailId)), from(T), T.itemId === item).query[Ident].to[Set] + run(select(T.sentMailId.s), from(T), T.itemId === item).query[Ident].to[Set] def deleteAllByItem(item: Ident): ConnectionIO[Int] = DML.delete(T, T.itemId === item) diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala index 11b87270..25847d50 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -1,5 +1,7 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -41,7 +43,7 @@ object RSource { val fileFilter = Column[Glob]("file_filter", this) val all = - List( + NonEmptyList.of[Column[_]]( sid, cid, abbrev, @@ -123,7 +125,7 @@ object RSource { order: Table => Column[_] ): Fragment = { val t = RSource.as("s") - Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).run + Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).build } def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = diff --git a/modules/store/src/main/scala/docspell/store/records/RTag.scala b/modules/store/src/main/scala/docspell/store/records/RTag.scala index 577e6e5d..777bcc1a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTag.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala @@ -27,7 +27,7 @@ object RTag { val name = Column[String]("name", this) val category = Column[String]("category", this) val created = Column[Timestamp]("created", this) - val all = List[Column[_]](tid, cid, name, category, created) + val all = NonEmptyList.of[Column[_]](tid, cid, name, category, created) } val T = Table(None) def as(alias: String): Table = @@ -75,7 +75,7 @@ object RTag { val nameFilter = nameQ.map(s => T.name.like(s"%${s.toLowerCase}%")) val sql = Select(select(T.all), from(T), T.cid === coll &&? nameFilter).orderBy(order(T)) - sql.run.query[RTag].to[Vector] + sql.build.query[RTag].to[Vector] } def findAllById(ids: List[Ident]): ConnectionIO[Vector[RTag]] = @@ -97,7 +97,7 @@ object RTag { from(t).innerJoin(ti, ti.tagId === t.tid), ti.itemId === itemId ).orderBy(t.name.asc) - sql.run.query[RTag].to[Vector] + sql.build.query[RTag].to[Vector] } def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = { @@ -109,7 +109,7 @@ object RTag { from(t).innerJoin(s, s.tagId === t.tid), s.sourceId === source ).orderBy(t.name.asc) - sql.run.query[RTag].to[Vector] + sql.build.query[RTag].to[Vector] } def findAllByNameOrId( diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index b702beb3..5e9f4eb2 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -19,7 +19,7 @@ object RTagItem { val tagItemId = Column[Ident]("tagitemid", this) val itemId = Column[Ident]("itemid", this) val tagId = Column[Ident]("tid", this) - val all = List(tagItemId, itemId, tagId) + val all = NonEmptyList.of[Column[_]](tagItemId, itemId, tagId) } val t = Table(None) def as(alias: String): Table = @@ -31,16 +31,8 @@ object RTagItem { def deleteItemTags(item: Ident): ConnectionIO[Int] = DML.delete(t, t.itemId === item) - def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] = { - print(cid) - DML.delete(t, t.itemId.in(items)) - //TODO match those of the collective - //val itemsFiltered = - // RItem.filterItemsFragment(items, cid) - //val sql = fr"DELETE FROM" ++ Fragment.const(t.tableName) ++ fr"WHERE" ++ - // t.itemId.isIn(itemsFiltered) - //sql.update.run - } + def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] = + DML.delete(t, t.itemId.in(RItem.filterItemsFragment(items, cid))) def deleteTag(tid: Ident): ConnectionIO[Int] = DML.delete(t, t.tagId === tid) diff --git a/modules/store/src/main/scala/docspell/store/records/RTagSource.scala b/modules/store/src/main/scala/docspell/store/records/RTagSource.scala index 8558c98a..86f6faff 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagSource.scala @@ -1,5 +1,6 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ @@ -19,7 +20,7 @@ object RTagSource { val id = Column[Ident]("id", this) val sourceId = Column[Ident]("source_id", this) val tagId = Column[Ident]("tag_id", this) - val all = List(id, sourceId, tagId) + val all = NonEmptyList.of[Column[_]](id, sourceId, tagId) } private val t = Table(None) diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index 2862f729..befd4865 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -1,5 +1,7 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -34,7 +36,17 @@ object RUser { val created = Column[Timestamp]("created", this) val all = - List(uid, login, cid, password, state, email, loginCount, lastLogin, created) + NonEmptyList.of[Column[_]]( + uid, + login, + cid, + password, + state, + email, + loginCount, + lastLogin, + created + ) } def as(alias: String): Table = @@ -83,7 +95,7 @@ object RUser { def findAll(coll: Ident, order: Table => Column[_]): ConnectionIO[Vector[RUser]] = { val t = Table(None) - val sql = Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).run + val sql = Select(select(t.all), from(t), t.cid === coll).orderBy(order(t)).build sql.query[RUser].to[Vector] } diff --git a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala index 17935f08..ca9cf28a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserEmail.scala @@ -1,6 +1,6 @@ package docspell.store.records -import cats.data.OptionT +import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ @@ -118,7 +118,7 @@ object RUserEmail { val mailReplyTo = Column[MailAddress]("mail_replyto", this) val created = Column[Timestamp]("created", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, uid, name, @@ -188,7 +188,7 @@ object RUserEmail { user.cid === accId.collective && user.login === accId.user &&? nameFilter ).orderBy(email.name) - sql.run.query[RUserEmail] + sql.build.query[RUserEmail] } def findByAccount( diff --git a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala index d40b9c03..d7e9edc0 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala @@ -1,6 +1,6 @@ package docspell.store.records -import cats.data.OptionT +import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ @@ -106,7 +106,7 @@ object RUserImap { val imapCertCheck = Column[Boolean]("imap_certcheck", this) val created = Column[Timestamp]("created", this) - val all = List( + val all = NonEmptyList.of[Column[_]]( id, uid, name, @@ -173,7 +173,7 @@ object RUserImap { select(m.all), from(m).innerJoin(u, m.uid === u.uid), u.cid === accId.collective && u.login === accId.user &&? nameFilter - ).orderBy(m.name).run + ).orderBy(m.name).build sql.query[RUserImap] } diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala index f13550a7..e35c73d9 100644 --- a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -1,7 +1,6 @@ package docspell.store.qb import minitest._ -import docspell.store.qb._ import docspell.store.qb.model._ import docspell.store.qb.DSL._ @@ -31,9 +30,16 @@ object QueryBuilderTest extends SimpleTestSuite { val q = Select(proj, tables, cond).orderBy(c.name.desc) q match { - case Select.Ordered(Select.SimpleSelect(proj, from, where, group), sb, vempty) => + case Select.Ordered( + Select.SimpleSelect(false, proj, from, where, group), + sb, + vempty + ) => assert(vempty.isEmpty) - assertEquals(sb, OrderBy(SelectExpr.SelectColumn(c.name), OrderBy.OrderType.Desc)) + assertEquals( + sb, + OrderBy(SelectExpr.SelectColumn(c.name, None), OrderBy.OrderType.Desc) + ) assertEquals(11, proj.size) from match { case FromExpr.From(_) => @@ -55,6 +61,8 @@ object QueryBuilderTest extends SimpleTestSuite { case _ => fail("Unexpected join result") } + case _ => + fail("Unexpected result") } assertEquals(group, None) assert(where.isDefined) diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/SelectBuilderTest.scala similarity index 90% rename from modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala rename to modules/store/src/test/scala/docspell/store/qb/impl/SelectBuilderTest.scala index 2d920352..f3fc8a9e 100644 --- a/modules/store/src/test/scala/docspell/store/qb/impl/DoobieQueryTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/impl/SelectBuilderTest.scala @@ -5,7 +5,7 @@ import docspell.store.qb._ import docspell.store.qb.model._ import docspell.store.qb.DSL._ -object DoobieQueryTest extends SimpleTestSuite { +object SelectBuilderTest extends SimpleTestSuite { test("basic fragment") { val c = CourseRecord.as("c") @@ -25,7 +25,7 @@ object DoobieQueryTest extends SimpleTestSuite { val frag = SelectBuilder(q) assertEquals( frag.toString, - """Fragment("SELECT c.id, c.name, c.owner_id, c.lecturer_id, c.lessons FROM course c INNER JOIN person o ON c.owner_id = o.id LEFT JOIN person l ON c.lecturer_id = l.id WHERE (LOWER(c.name) LIKE ? AND o.name = ? )")""" + """Fragment("SELECT c.id, c.name, c.owner_id, c.lecturer_id, c.lessons FROM course c INNER JOIN person o ON c.owner_id = o.id LEFT JOIN person l ON c.lecturer_id = l.id WHERE (LOWER(c.name) LIKE ? AND o.name = ? )")""" ) } diff --git a/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala index 2024fd1f..6b53fdfb 100644 --- a/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala +++ b/modules/store/src/test/scala/docspell/store/qb/model/CourseRecord.scala @@ -1,5 +1,6 @@ package docspell.store.qb.model +import cats.data.NonEmptyList import docspell.store.qb._ case class CourseRecord( @@ -22,7 +23,7 @@ object CourseRecord { val lecturerId = Column[Long]("lecturer_id", this) val lessons = Column[Int]("lessons", this) - val all = List(id, name, ownerId, lecturerId, lessons) + val all = NonEmptyList.of[Column[_]](id, name, ownerId, lecturerId, lessons) } def as(alias: String): Table = diff --git a/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala index a328c6a8..5ea5b653 100644 --- a/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala +++ b/modules/store/src/test/scala/docspell/store/qb/model/PersonRecord.scala @@ -1,5 +1,6 @@ package docspell.store.qb.model +import cats.data.NonEmptyList import docspell.store.qb._ import docspell.common._ @@ -15,7 +16,7 @@ object PersonRecord { val name = Column[String]("name", this) val created = Column[Timestamp]("created", this) - val all = List(id, name, created) + val all = NonEmptyList.of[Column[_]](id, name, created) } def as(alias: String): Table = From fd6d09587d0dfd6da5303d6d9bc743eb026c56fd Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 13:35:40 +0100 Subject: [PATCH 14/38] Convert more records --- .../main/scala/docspell/store/qb/DML.scala | 4 + .../main/scala/docspell/store/qb/DSL.scala | 6 + .../main/scala/docspell/store/qb/Setter.scala | 1 + .../docspell/store/queries/QAttachment.scala | 9 +- .../scala/docspell/store/queries/QItem.scala | 79 +-- .../scala/docspell/store/queries/QLogin.scala | 26 +- .../docspell/store/records/RAttachment.scala | 477 +++++++++++------- .../store/records/RAttachmentArchive.scala | 171 ++++--- .../store/records/RAttachmentMeta.scala | 82 +-- .../store/records/RAttachmentPreview.scala | 174 ++++--- .../store/records/RAttachmentSource.scala | 154 ++++-- .../store/records/RClassifierSetting.scala | 95 ++-- .../docspell/store/records/RCollective.scala | 149 +++--- .../docspell/store/records/RFileMeta.scala | 38 +- .../scala/docspell/store/records/RItem.scala | 206 ++++---- 15 files changed, 996 insertions(+), 675 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/DML.scala b/modules/store/src/main/scala/docspell/store/qb/DML.scala index 48a7f051..c30e790d 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DML.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DML.scala @@ -74,6 +74,10 @@ object DML { case Setter.Increment(column, amount) => val colFrag = SelectExprBuilder.columnNoPrefix(column) colFrag ++ fr" =" ++ colFrag ++ fr" + $amount" + + case Setter.Decrement(column, amount) => + val colFrag = SelectExprBuilder.columnNoPrefix(column) + colFrag ++ fr" =" ++ colFrag ++ fr" - $amount" } def set(s: Setter[_], more: Setter[_]*): Nel[Setter[_]] = diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 7e388075..dbddbdc9 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -127,6 +127,9 @@ trait DSL extends DoobieMeta { def increment(amount: Int): Setter[A] = Setter.Increment(col, amount) + def decrement(amount: Int): Setter[A] = + Setter.Decrement(col, amount) + def asc: OrderBy = OrderBy(SelectExpr.SelectColumn(col, None), OrderBy.OrderType.Asc) @@ -177,6 +180,9 @@ trait DSL extends DoobieMeta { def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) + + def <>(other: Column[A]): Condition = + Condition.CompareCol(col, Operator.Neq, other) } implicit final class ConditionOps(c: Condition) { diff --git a/modules/store/src/main/scala/docspell/store/qb/Setter.scala b/modules/store/src/main/scala/docspell/store/qb/Setter.scala index d86af800..b6808e55 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Setter.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Setter.scala @@ -13,5 +13,6 @@ object Setter { extends Setter[A] case class Increment[A](column: Column[A], amount: Int) extends Setter[A] + case class Decrement[A](column: Column[A], amount: Int) extends Setter[A] } diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 86ae26f4..f1aae89a 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -180,14 +180,17 @@ object QAttachment { val iId = RItem.Columns.id.prefix("i") val iColl = RItem.Columns.cid.prefix("i") val iFolder = RItem.Columns.folder.prefix("i") - val cId = RCollective.Columns.id.prefix("c") - val cLang = RCollective.Columns.language.prefix("c") + val c = RCollective.as("c") + val cId = c.id.column + val cLang = c.language.column val cols = Seq(aId, aItem, iColl, iFolder, cLang, aName, mContent) val from = RAttachment.table ++ fr"a INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ - fr"INNER JOIN" ++ RCollective.table ++ fr"c ON" ++ cId.is(iColl) + fr"INNER JOIN" ++ Fragment.const(RCollective.T.tableName) ++ fr"c ON" ++ cId.is( + iColl + ) val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index da2981e9..81bb78b0 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -633,27 +633,31 @@ object QItem { limit: Option[Int], states: Set[ItemState] ): Fragment = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iState = RItem.Columns.state.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1Id = RFileMeta.Columns.id.prefix("m1") - val m2Id = RFileMeta.Columns.id.prefix("m2") - val m3Id = RFileMeta.Columns.id.prefix("m3") + val IC = RItem.Columns.all.map(_.prefix("i")) + val aItem = RAttachment.Columns.itemId.prefix("a") + val aId = RAttachment.Columns.id.prefix("a") + val aFileId = RAttachment.Columns.fileId.prefix("a") + val iId = RItem.Columns.id.prefix("i") + val iState = RItem.Columns.state.prefix("i") + val sId = RAttachmentSource.Columns.id.prefix("s") + val sFileId = RAttachmentSource.Columns.fileId.prefix("s") + val rId = RAttachmentArchive.Columns.id.prefix("r") + val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") + val m1 = RFileMeta.as("m1") + val m2 = RFileMeta.as("m2") + val m3 = RFileMeta.as("m3") + val m1Id = m1.id.column + val m2Id = m2.id.column + val m3Id = m3.id.column + val filemetaTable = Fragment.const(RFileMeta.T.tableName) val from = RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ RFileMeta.table ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ RFileMeta.table ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ + fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ + fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ RFileMeta.table ++ fr"m3 ON" ++ m3Id.is(rFileId) + fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) val fileCond = or(m1Id.isIn(fileMetaIds), m2Id.isIn(fileMetaIds), m3Id.isIn(fileMetaIds)) @@ -691,30 +695,33 @@ object QItem { } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1Id = RFileMeta.Columns.id.prefix("m1") - val m2Id = RFileMeta.Columns.id.prefix("m2") - val m3Id = RFileMeta.Columns.id.prefix("m3") - val m1Checksum = RFileMeta.Columns.checksum.prefix("m1") - val m2Checksum = RFileMeta.Columns.checksum.prefix("m2") - val m3Checksum = RFileMeta.Columns.checksum.prefix("m3") - + val IC = RItem.Columns.all.map(_.prefix("i")) + val aItem = RAttachment.Columns.itemId.prefix("a") + val aId = RAttachment.Columns.id.prefix("a") + val aFileId = RAttachment.Columns.fileId.prefix("a") + val iId = RItem.Columns.id.prefix("i") + val iColl = RItem.Columns.cid.prefix("i") + val sId = RAttachmentSource.Columns.id.prefix("s") + val sFileId = RAttachmentSource.Columns.fileId.prefix("s") + val rId = RAttachmentArchive.Columns.id.prefix("r") + val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") + val m1 = RFileMeta.as("m1") + val m2 = RFileMeta.as("m2") + val m3 = RFileMeta.as("m3") + val m1Id = m1.id.column + val m2Id = m2.id.column + val m3Id = m3.id.column + val m1Checksum = m1.checksum.column + val m2Checksum = m2.checksum.column + val m3Checksum = m3.checksum.column + val filemetaTable = Fragment.const(RFileMeta.T.tableName) val from = RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ RFileMeta.table ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ RFileMeta.table ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ + fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ + fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ RFileMeta.table ++ fr"m3 ON" ++ m3Id.is(rFileId) + fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) selectSimple( IC, diff --git a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala index 7dfdf59f..08af3265 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala @@ -3,8 +3,8 @@ package docspell.store.queries import cats.data.OptionT import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.records.RCollective.{Columns => CC} +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records.{RCollective, RRememberMe, RUser} import doobie._ @@ -22,20 +22,14 @@ object QLogin { ) def findUser(acc: AccountId): ConnectionIO[Option[Data]] = { - val user = RUser.as("u") - val ucid = user.cid.column - val login = user.login.column - val pass = user.password.column - val ustate = user.state.column - val cstate = CC.state.prefix("c") - val ccid = CC.id.prefix("c") - - val sql = selectSimple( - List(ucid, login, pass, cstate, ustate), - Fragment.const(user.tableName) ++ fr"u, " ++ RCollective.table ++ fr"c", - and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective)) - ) - + val user = RUser.as("u") + val coll = RCollective.as("c") + val sql = + Select( + select(user.cid, user.login, user.password, coll.state, user.state), + from(user).innerJoin(coll, user.cid === coll.id), + user.login === acc.user && user.cid === acc.collective + ).build logger.trace(s"SQL : $sql") sql.query[Data].option } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index e9d5d935..26372748 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -5,8 +5,9 @@ import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb.TableDef +import docspell.store.qb._ import bitpeace.FileMeta import doobie._ @@ -22,10 +23,27 @@ case class RAttachment( ) {} object RAttachment { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "attachment" + + val id = Column[Ident]("attachid", this) + val itemId = Column[Ident]("itemid", this) + val fileId = Column[Ident]("filemetaid", this) + val position = Column[Int]("position", this) + val created = Column[Timestamp]("created", this) + val name = Column[String]("name", this) + val all = NonEmptyList.of[Column[_]](id, itemId, fileId, position, created, name) + } + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) val table = fr"attachment" object Columns { + import docspell.store.impl._ + val id = Column("attachid") val itemId = Column("itemid") val fileId = Column("filemetaid") @@ -34,32 +52,37 @@ object RAttachment { val name = Column("name") val all = List(id, itemId, fileId, position, created, name) } - import Columns._ def insert(v: RAttachment): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}" - ).update.run + ) def decPositions(iId: Ident, lowerBound: Int, upperBound: Int): ConnectionIO[Int] = - updateRow( - table, - and(itemId.is(iId), position.isGte(lowerBound), position.isLte(upperBound)), - position.decrement(1) - ).update.run + DML.update( + T, + where( + T.itemId === iId && T.position >= lowerBound && T.position <= upperBound + ), + DML.set(T.position.decrement(1)) + ) def incPositions(iId: Ident, lowerBound: Int, upperBound: Int): ConnectionIO[Int] = - updateRow( - table, - and(itemId.is(iId), position.isGte(lowerBound), position.isLte(upperBound)), - position.increment(1) - ).update.run + DML.update( + T, + where( + T.itemId === iId && T.position >= lowerBound && T.position <= upperBound + ), + DML.set(T.position.increment(1)) + ) def nextPosition(id: Ident): ConnectionIO[Int] = for { - max <- selectSimple(position.max, table, itemId.is(id)).query[Option[Int]].unique + max <- Select(max(T.position).s, from(T), T.itemId === id).build + .query[Option[Int]] + .unique } yield max.map(_ + 1).getOrElse(0) def updateFileIdAndName( @@ -67,41 +90,49 @@ object RAttachment { fId: Ident, fname: Option[String] ): ConnectionIO[Int] = - updateRow( - table, - id.is(attachId), - commas(fileId.setTo(fId), name.setTo(fname)) - ).update.run + DML.update( + T, + T.id === attachId, + DML.set(T.fileId.setTo(fId), T.name.setTo(fname)) + ) def updateFileId( attachId: Ident, fId: Ident ): ConnectionIO[Int] = - updateRow( - table, - id.is(attachId), - fileId.setTo(fId) - ).update.run + DML.update( + T, + T.id === attachId, + DML.set(T.fileId.setTo(fId)) + ) def updatePosition(attachId: Ident, pos: Int): ConnectionIO[Int] = - updateRow(table, id.is(attachId), position.setTo(pos)).update.run + DML.update(T, T.id === attachId, DML.set(T.position.setTo(pos))) def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] = - selectSimple(all, table, id.is(attachId)).query[RAttachment].option + run(select(T.all), from(T), T.id === attachId).query[RAttachment].option def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = { import bitpeace.sql._ - val cols = RFileMeta.Columns.all.map(_.prefix("m")) - val aId = id.prefix("a") - val aFileMeta = fileId.prefix("a") - val mId = RFileMeta.Columns.id.prefix("m") - - val from = - table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId) - val cond = aId.is(attachId) - - selectSimple(cols, from, cond).query[FileMeta].option +// val cols = RFileMeta.Columns.all.map(_.prefix("m")) +// val aId = id.prefix("a") +// val aFileMeta = fileId.prefix("a") +// val mId = RFileMeta.Columns.id.prefix("m") +// +// val from = +// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId) +// val cond = aId.is(attachId) +// +// selectSimple(cols, from, cond).query[FileMeta].option + val m = RFileMeta.as("m") + val a = RAttachment.as("a") + Select( + select(m.all), + from(a) + .innerJoin(m, a.fileId === m.id), + a.id === attachId + ).build.query[FileMeta].option } def updateName( @@ -109,7 +140,7 @@ object RAttachment { collective: Ident, aname: Option[String] ): ConnectionIO[Int] = { - val update = updateRow(table, id.is(attachId), name.setTo(aname)).update.run + val update = DML.update(T, T.id === attachId, DML.set(T.name.setTo(aname))) for { exists <- existsByIdAndCollective(attachId, collective) n <- if (exists) update else 0.pure[ConnectionIO] @@ -119,44 +150,59 @@ object RAttachment { def findByIdAndCollective( attachId: Ident, collective: Ident - ): ConnectionIO[Option[RAttachment]] = - selectSimple( - all.map(_.prefix("a")), - table ++ fr"a," ++ RItem.table ++ fr"i", - and( - fr"a.itemid = i.itemid", - id.prefix("a").is(attachId), - RItem.Columns.cid.prefix("i").is(collective) - ) - ).query[RAttachment].option + ): ConnectionIO[Option[RAttachment]] = { + val a = RAttachment.as("a") + val i = RItem.as("i") + Select( + select(a.all), + from(a).innerJoin(i, a.itemId === i.id), + a.id === attachId && i.cid === collective + ).build.query[RAttachment].option + } def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] = - selectSimple(all, table, itemId.is(id)).query[RAttachment].to[Vector] + run(select(T.all), from(T), T.itemId === id).query[RAttachment].to[Vector] def existsByIdAndCollective( attachId: Ident, collective: Ident ): ConnectionIO[Boolean] = { - val aId = id.prefix("a") - val aItem = itemId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val from = - table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) - val cond = and(iColl.is(collective), aId.is(attachId)) - selectCount(id, from, cond).query[Int].unique.map(_ > 0) +// val aId = id.prefix("a") +// val aItem = itemId.prefix("a") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// val from = +// table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) +// val cond = and(iColl.is(collective), aId.is(attachId)) +// selectCount(id, from, cond).query[Int].unique.map(_ > 0) + val a = RAttachment.as("a") + val i = RItem.as("i") + Select( + count(a.id).s, + from(a) + .innerJoin(i, a.itemId === i.id), + i.cid === collective && a.id === attachId + ).build.query[Int].unique.map(_ > 0) } def findByItemAndCollective( id: Ident, coll: Ident ): ConnectionIO[Vector[RAttachment]] = { - val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id - .prefix("i") - .is(itemId.prefix("a")) ++ - fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) - q.query[RAttachment].to[Vector] +// val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id +// .prefix("i") +// .is(itemId.prefix("a")) ++ +// fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) +// q.query[RAttachment].to[Vector] + val a = RAttachment.as("a") + val i = RItem.as("i") + Select( + select(a.all), + from(a) + .innerJoin(i, i.id === a.itemId), + a.itemId === id && i.cid === coll + ).build.query[RAttachment].to[Vector] } def findByItemCollectiveSource( @@ -165,28 +211,42 @@ object RAttachment { fileIds: NonEmptyList[Ident] ): ConnectionIO[Vector[RAttachment]] = { - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val aItem = Columns.itemId.prefix("a") - val aId = Columns.id.prefix("a") - val aFile = Columns.fileId.prefix("a") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFile = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFile = RAttachmentArchive.Columns.fileId.prefix("r") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// val aItem = Columns.itemId.prefix("a") +// val aId = Columns.id.prefix("a") +// val aFile = Columns.fileId.prefix("a") +// val sId = RAttachmentSource.Columns.id.prefix("s") +// val sFile = RAttachmentSource.Columns.fileId.prefix("s") +// val rId = RAttachmentArchive.Columns.id.prefix("r") +// val rFile = RAttachmentArchive.Columns.fileId.prefix("r") +// +// val from = table ++ fr"a INNER JOIN" ++ +// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"LEFT JOIN" ++ +// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"LEFT JOIN" ++ +// RAttachmentArchive.table ++ fr"r ON" ++ rId.is(aId) +// +// val cond = and( +// iId.is(id), +// iColl.is(coll), +// or(aFile.isIn(fileIds), sFile.isIn(fileIds), rFile.isIn(fileIds)) +// ) +// +// selectSimple(all.map(_.prefix("a")), from, cond).query[RAttachment].to[Vector] + val i = RItem.as("i") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val r = RAttachmentArchive.as("r") - val from = table ++ fr"a INNER JOIN" ++ - RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"LEFT JOIN" ++ - RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"LEFT JOIN" ++ - RAttachmentArchive.table ++ fr"r ON" ++ rId.is(aId) - - val cond = and( - iId.is(id), - iColl.is(coll), - or(aFile.isIn(fileIds), sFile.isIn(fileIds), rFile.isIn(fileIds)) - ) - - selectSimple(all.map(_.prefix("a")), from, cond).query[RAttachment].to[Vector] + Select( + select(a.all), + from(a) + .innerJoin(i, i.id === a.itemId) + .leftJoin(s, s.id === a.id) + .leftJoin(r, r.id === a.id), + i.id === id && i.cid === coll && + (a.fileId.in(fileIds) || s.fileId.in(fileIds) || r.fileId.in(fileIds)) + ).build.query[RAttachment].to[Vector] } def findByItemAndCollectiveWithMeta( @@ -195,27 +255,45 @@ object RAttachment { ): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ - val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) - val afileMeta = fileId.prefix("a") - val aItem = itemId.prefix("a") - val mId = RFileMeta.Columns.id.prefix("m") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - - val from = - table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) - val cond = Seq(aItem.is(id), iColl.is(coll)) - - selectSimple(cols, from, and(cond)).query[(RAttachment, FileMeta)].to[Vector] +// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) +// val afileMeta = fileId.prefix("a") +// val aItem = itemId.prefix("a") +// val mId = RFileMeta.Columns.id.prefix("m") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = +// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) +// val cond = Seq(aItem.is(id), iColl.is(coll)) +// +// selectSimple(cols, from, and(cond)).query[(RAttachment, FileMeta)].to[Vector] + val a = RAttachment.as("a") + val m = RFileMeta.as("m") + val i = RItem.as("i") + Select( + select(a.all, m.all), + from(a) + .innerJoin(m, a.fileId === m.id) + .innerJoin(i, a.itemId === i.id), + a.itemId === id && i.cid === coll + ).build.query[(RAttachment, FileMeta)].to[Vector] } def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ - val q = - fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" - q.query[(RAttachment, FileMeta)].to[Vector] +// val q = +// fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filzemeta m +// WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" + val a = RAttachment.as("a") + val m = RFileMeta.as("m") + Select( + select(a.all, m.all), + from(a) + .innerJoin(m, a.fileId === m.id), + a.itemId === id + ).orderBy(a.position.asc).build.query[(RAttachment, FileMeta)].to[Vector] } /** Deletes the attachment and its related source and meta records. @@ -225,110 +303,159 @@ object RAttachment { n0 <- RAttachmentMeta.delete(attachId) n1 <- RAttachmentSource.delete(attachId) n2 <- RAttachmentPreview.delete(attachId) - n3 <- deleteFrom(table, id.is(attachId)).update.run + n3 <- DML.delete(T, T.id === attachId) } yield n0 + n1 + n2 + n3 def findItemId(attachId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(Seq(itemId), table, id.is(attachId)).query[Ident].option + Select(T.itemId.s, from(T), T.id === attachId).build.query[Ident].option def findAll( coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { - val aItem = Columns.itemId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - - val cols = all.map(_.prefix("a")) +// val aItem = Columns.itemId.prefix("a") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val cols = all.map(_.prefix("a")) +// +// coll match { +// case Some(cid) => +// val join = table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) +// val cond = iColl.is(cid) +// selectSimple(cols, join, cond) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) +// case None => +// selectSimple(cols, table, Fragment.empty) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) +// } + val a = RAttachment.as("a") + val i = RItem.as("i") coll match { case Some(cid) => - val join = table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) - val cond = iColl.is(cid) - selectSimple(cols, join, cond) - .query[RAttachment] - .streamWithChunkSize(chunkSize) + Select( + select(a.all), + from(a) + .innerJoin(i, i.id === a.itemId), + i.cid === cid + ).build.query[RAttachment].streamWithChunkSize(chunkSize) case None => - selectSimple(cols, table, Fragment.empty) + Select(select(a.all), from(a)).build .query[RAttachment] .streamWithChunkSize(chunkSize) } } def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = { - val aId = Columns.id.prefix("a") - val aCreated = Columns.created.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mPages = RAttachmentMeta.Columns.pages.prefix("m") - - val cols = all.map(_.prefix("a")) - val join = table ++ fr"a LEFT OUTER JOIN" ++ - RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) - val cond = mPages.isNull - - (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) - .query[RAttachment] - .streamWithChunkSize(chunkSize) +// val aId = Columns.id.prefix("a") +// val aCreated = Columns.created.prefix("a") +// val mId = RAttachmentMeta.Columns.id.prefix("m") +// val mPages = RAttachmentMeta.Columns.pages.prefix("m") +// +// val cols = all.map(_.prefix("a")) +// val join = table ++ fr"a LEFT OUTER JOIN" ++ +// RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) +// val cond = mPages.isNull +// +// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) + val a = RAttachment.as("a") + val m = RAttachmentMeta.as("m") + Select( + select(a.all), + from(a) + .leftJoin(m, a.id === m.id), + m.pages.isNull + ).build.query[RAttachment].streamWithChunkSize(chunkSize) } def findWithoutPreview( coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { - val aId = Columns.id.prefix("a") - val aItem = Columns.itemId.prefix("a") - val aCreated = Columns.created.prefix("a") - val pId = RAttachmentPreview.Columns.id.prefix("p") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") +// val aId = Columns.id.prefix("a") +// val aItem = Columns.itemId.prefix("a") +// val aCreated = Columns.created.prefix("a") +// val pId = RAttachmentPreview.Columns.id.prefix("p") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val cols = all.map(_.prefix("a")) +// val baseJoin = +// table ++ fr"a LEFT OUTER JOIN" ++ +// RAttachmentPreview.table ++ fr"p ON" ++ pId.is(aId) +// +// val baseCond = +// Seq(pId.isNull) +// +// coll match { +// case Some(cid) => +// val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) +// val cond = and(baseCond ++ Seq(iColl.is(cid))) +// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) +// case None => +// (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc)) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) +// } + val a = RAttachment.as("a") + val p = RAttachmentPreview.as("p") + val i = RItem.as("i") - val cols = all.map(_.prefix("a")) - val baseJoin = - table ++ fr"a LEFT OUTER JOIN" ++ - RAttachmentPreview.table ++ fr"p ON" ++ pId.is(aId) - - val baseCond = - Seq(pId.isNull) - - coll match { - case Some(cid) => - val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) - val cond = and(baseCond ++ Seq(iColl.is(cid))) - (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) - .query[RAttachment] - .streamWithChunkSize(chunkSize) - case None => - (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc)) - .query[RAttachment] - .streamWithChunkSize(chunkSize) - } + val baseJoin = from(a).leftJoin(p, p.id === a.id) + Select( + select(a.all), + coll.map(_ => baseJoin.innerJoin(i, i.id === a.itemId)).getOrElse(baseJoin), + p.id.isNull &&? coll.map(cid => i.cid === cid) + ).orderBy(a.created.asc).build.query[RAttachment].streamWithChunkSize(chunkSize) } def findNonConvertedPdf( coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { - val aId = Columns.id.prefix("a") - val aItem = Columns.itemId.prefix("a") - val aFile = Columns.fileId.prefix("a") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFile = RAttachmentSource.Columns.fileId.prefix("s") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val mId = RFileMeta.Columns.id.prefix("m") - val mType = RFileMeta.Columns.mimetype.prefix("m") +// val aId = Columns.id.prefix("a") +// val aItem = Columns.itemId.prefix("a") +// val aFile = Columns.fileId.prefix("a") +// val sId = RAttachmentSource.Columns.id.prefix("s") +// val sFile = RAttachmentSource.Columns.fileId.prefix("s") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// val mId = RFileMeta.Columns.id.prefix("m") +// val mType = RFileMeta.Columns.mimetype.prefix("m") val pdfType = "application/pdf%" +// +// val from = table ++ fr"a INNER JOIN" ++ +// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"INNER JOIN" ++ +// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"INNER JOIN" ++ +// RFileMeta.table ++ fr"m ON" ++ aFile.is(mId) +// val where = coll match { +// case Some(cid) => and(iColl.is(cid), aFile.is(sFile), mType.lowerLike(pdfType)) +// case None => and(aFile.is(sFile), mType.lowerLike(pdfType)) +// } +// selectSimple(all.map(_.prefix("a")), from, where) +// .query[RAttachment] +// .streamWithChunkSize(chunkSize) + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val i = RItem.as("i") + val m = RFileMeta.as("m") - val from = table ++ fr"a INNER JOIN" ++ - RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"INNER JOIN" ++ - RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"INNER JOIN" ++ - RFileMeta.table ++ fr"m ON" ++ aFile.is(mId) - val where = coll match { - case Some(cid) => and(iColl.is(cid), aFile.is(sFile), mType.lowerLike(pdfType)) - case None => and(aFile.is(sFile), mType.lowerLike(pdfType)) - } - selectSimple(all.map(_.prefix("a")), from, where) - .query[RAttachment] - .streamWithChunkSize(chunkSize) + Select( + select(a.all), + from(a) + .innerJoin(s, s.id === a.id) + .innerJoin(i, i.id === a.itemId) + .innerJoin(m, m.id === a.fileId), + a.fileId === s.fileId && + m.mimetype.likes(pdfType) &&? + coll.map(cid => i.cid === cid) + ).build.query[RAttachment].streamWithChunkSize(chunkSize) } } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala index 917741c3..ab97b9b2 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala @@ -3,8 +3,9 @@ package docspell.store.records import cats.data.NonEmptyList import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb.TableDef +import docspell.store.qb._ import bitpeace.FileMeta import doobie._ @@ -22,10 +23,25 @@ case class RAttachmentArchive( ) object RAttachmentArchive { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "attachment_archive" + + val id = Column[Ident]("id", this) + val fileId = Column[Ident]("file_id", this) + val name = Column[String]("filename", this) + val messageId = Column[String]("message_id", this) + val created = Column[Timestamp]("created", this) + + val all = NonEmptyList.of[Column[_]](id, fileId, name, messageId, created) + } + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) val table = fr"attachment_archive" - object Columns { + import docspell.store.impl._ + val id = Column("id") val fileId = Column("file_id") val name = Column("filename") @@ -35,64 +51,83 @@ object RAttachmentArchive { val all = List(id, fileId, name, messageId, created) } - import Columns._ - def of(ra: RAttachment, mId: Option[String]): RAttachmentArchive = RAttachmentArchive(ra.id, ra.fileId, ra.name, mId, ra.created) def insert(v: RAttachmentArchive): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.fileId},${v.name},${v.messageId},${v.created}" - ).update.run + ) def findById(attachId: Ident): ConnectionIO[Option[RAttachmentArchive]] = - selectSimple(all, table, id.is(attachId)).query[RAttachmentArchive].option + run(select(T.all), from(T), T.id === attachId).query[RAttachmentArchive].option def delete(attachId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(attachId)).update.run + DML.delete(T, T.id === attachId) def deleteAll(fId: Ident): ConnectionIO[Int] = - deleteFrom(table, fileId.is(fId)).update.run + DML.delete(T, T.fileId === fId) def findByIdAndCollective( attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentArchive]] = { - val bId = RAttachment.Columns.id.prefix("b") - val aId = Columns.id.prefix("a") - val bItem = RAttachment.Columns.itemId.prefix("b") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") +// val bId = RAttachment.Columns.id.prefix("b") +// val aId = Columns.id.prefix("a") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = table ++ fr"a INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) +// +// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) +// +// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].option + val b = RAttachment.as("b") + val a = RAttachmentArchive.as("a") + val i = RItem.as("i") - val from = table ++ fr"a INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) - - val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) - - selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].option + Select( + select(a.all), + from(a) + .innerJoin(b, b.id === a.id) + .innerJoin(i, i.id === b.itemId), + a.id === attachId && b.id === attachId && i.cid === collective + ).build.query[RAttachmentArchive].option } def findByMessageIdAndCollective( messageIds: NonEmptyList[String], collective: Ident ): ConnectionIO[Vector[RAttachmentArchive]] = { - val bId = RAttachment.Columns.id.prefix("b") - val bItem = RAttachment.Columns.itemId.prefix("b") - val aMsgId = Columns.messageId.prefix("a") - val aId = Columns.id.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - - val from = table ++ fr"a INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) - - val where = and(aMsgId.isIn(messageIds), iColl.is(collective)) - - selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].to[Vector] +// val bId = RAttachment.Columns.id.prefix("b") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val aMsgId = Columns.messageId.prefix("a") +// val aId = Columns.id.prefix("a") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = table ++ fr"a INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) +// +// val where = and(aMsgId.isIn(messageIds), iColl.is(collective)) +// +// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].to[Vector] + val b = RAttachment.as("b") + val a = RAttachmentArchive.as("a") + val i = RItem.as("i") + Select( + select(a.all), + from(a) + .innerJoin(b, b.id === a.id) + .innerJoin(i, i.id === b.itemId), + a.messageId.in(messageIds) && i.cid === collective + ).build.query[RAttachmentArchive].to[Vector] } def findByItemWithMeta( @@ -100,31 +135,49 @@ object RAttachmentArchive { ): ConnectionIO[Vector[(RAttachmentArchive, FileMeta)]] = { import bitpeace.sql._ - val aId = Columns.id.prefix("a") - val afileMeta = fileId.prefix("a") - val bPos = RAttachment.Columns.position.prefix("b") - val bId = RAttachment.Columns.id.prefix("b") - val bItem = RAttachment.Columns.itemId.prefix("b") - val mId = RFileMeta.Columns.id.prefix("m") - - val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) - val from = table ++ fr"a INNER JOIN" ++ - RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) - val where = bItem.is(id) - - (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) - .query[(RAttachmentArchive, FileMeta)] - .to[Vector] +// val aId = Columns.id.prefix("a") +// val afileMeta = fileId.prefix("a") +// val bPos = RAttachment.Columns.position.prefix("b") +// val bId = RAttachment.Columns.id.prefix("b") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val mId = RFileMeta.as("m").id.column +// +// val cols = all.map(_.prefix("a")) ++ RFileMeta.as("m").all.map(_.column).toList +// val from = table ++ fr"a INNER JOIN" ++ +// Fragment.const(RFileMeta.T.tableName) ++ fr"m ON" ++ afileMeta.is( +// mId +// ) ++ fr"INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) +// val where = bItem.is(id) +// +// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) +// .query[(RAttachmentArchive, FileMeta)] +// .to[Vector] + val a = RAttachmentArchive.as("a") + val b = RAttachment.as("b") + val m = RFileMeta.as("m") + Select( + select(a.all, m.all), + from(a) + .innerJoin(m, a.fileId === m.id) + .innerJoin(b, a.id === b.id), + b.itemId === id + ).orderBy(b.position.asc).build.query[(RAttachmentArchive, FileMeta)].to[Vector] } /** If the given attachment id has an associated archive, this returns * the number of all associated attachments. Returns 0 if there is * no archive for the given attachment. */ - def countEntries(attachId: Ident): ConnectionIO[Int] = { - val qFileId = selectSimple(Seq(fileId), table, id.is(attachId)) - val q = selectCount(id, table, fileId.isSubquery(qFileId)) - q.query[Int].unique - } + def countEntries(attachId: Ident): ConnectionIO[Int] = +// val qFileId = selectSimple(Seq(fileId), table, id.is(attachId)) +// val q = selectCount(id, table, fileId.isSubquery(qFileId)) +// q.query[Int].unique + Select( + count(T.id).s, + from(T), + T.fileId.in(Select(T.fileId.s, from(T), T.id === attachId)) + ).build.query[Int].unique + //TODO this looks strange, can be simplified + } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 5fcd5b93..ad5558b8 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -1,10 +1,11 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -29,9 +30,25 @@ object RAttachmentMeta { def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty, None) - val table = fr"attachmentmeta" + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "attachmentmeta" + val id = Column[Ident]("attachid", this) + val content = Column[String]("content", this) + val nerlabels = Column[List[NerLabel]]("nerlabels", this) + val proposals = Column[MetaProposalList]("itemproposals", this) + val pages = Column[Int]("page_count", this) + val all = NonEmptyList.of[Column[_]](id, content, nerlabels, proposals, pages) + } + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + + val table = fr"attachmentmeta" object Columns { + import docspell.store.impl._ + val id = Column("attachid") val content = Column("content") val nerlabels = Column("nerlabels") @@ -39,23 +56,22 @@ object RAttachmentMeta { val pages = Column("page_count") val all = List(id, content, nerlabels, proposals, pages) } - import Columns._ def insert(v: RAttachmentMeta): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals},${v.pages}" - ).update.run + ) def exists(attachId: Ident): ConnectionIO[Boolean] = - selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0) + Select(count(T.id).s, from(T), T.id === attachId).build.query[Int].unique.map(_ > 0) def findById(attachId: Ident): ConnectionIO[Option[RAttachmentMeta]] = - selectSimple(all, table, id.is(attachId)).query[RAttachmentMeta].option + run(select(T.all), from(T), T.id === attachId).query[RAttachmentMeta].option def findPageCountById(attachId: Ident): ConnectionIO[Option[Int]] = - selectSimple(Seq(pages), table, id.is(attachId)) + Select(T.pages.s, from(T), T.id === attachId).build .query[Option[Int]] .option .map(_.flatten) @@ -67,37 +83,37 @@ object RAttachmentMeta { } yield n1 def update(v: RAttachmentMeta): ConnectionIO[Int] = - updateRow( - table, - id.is(v.id), - commas( - content.setTo(v.content), - nerlabels.setTo(v.nerlabels), - proposals.setTo(v.proposals) + DML.update( + T, + T.id === v.id, + DML.set( + T.content.setTo(v.content), + T.nerlabels.setTo(v.nerlabels), + T.proposals.setTo(v.proposals) ) - ).update.run + ) def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] = - updateRow( - table, - id.is(mid), - commas( - nerlabels.setTo(labels) + DML.update( + T, + T.id === mid, + DML.set( + T.nerlabels.setTo(labels) ) - ).update.run + ) def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] = - updateRow( - table, - id.is(mid), - commas( - proposals.setTo(plist) + DML.update( + T, + T.id === mid, + DML.set( + T.proposals.setTo(plist) ) - ).update.run + ) def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] = - updateRow(table, id.is(mid), pages.setTo(pageCount)).update.run + DML.update(T, T.id === mid, DML.set(T.pages.setTo(pageCount))) def delete(attachId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(attachId)).update.run + DML.delete(T, T.id === attachId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index c28169b7..290efd50 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -1,8 +1,10 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import bitpeace.FileMeta import doobie._ @@ -19,10 +21,24 @@ case class RAttachmentPreview( ) object RAttachmentPreview { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "attachment_preview" + + val id = Column[Ident]("id", this) + val fileId = Column[Ident]("file_id", this) + val name = Column[String]("filename", this) + val created = Column[Timestamp]("created", this) + + val all = NonEmptyList.of[Column[_]](id, fileId, name, created) + } + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) val table = fr"attachment_preview" - object Columns { + import docspell.store.impl._ val id = Column("id") val fileId = Column("file_id") val name = Column("filename") @@ -31,67 +47,98 @@ object RAttachmentPreview { val all = List(id, fileId, name, created) } - import Columns._ - def insert(v: RAttachmentPreview): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run + DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") def findById(attachId: Ident): ConnectionIO[Option[RAttachmentPreview]] = - selectSimple(all, table, id.is(attachId)).query[RAttachmentPreview].option + run(select(T.all), from(T), T.id === attachId).query[RAttachmentPreview].option def delete(attachId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(attachId)).update.run + DML.delete(T, T.id === attachId) def findByIdAndCollective( attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentPreview]] = { - val bId = RAttachment.Columns.id.prefix("b") - val aId = Columns.id.prefix("a") - val bItem = RAttachment.Columns.itemId.prefix("b") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") +// val bId = RAttachment.Columns.id.prefix("b") +// val aId = Columns.id.prefix("a") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = table ++ fr"a INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) +// +// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) +// +// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentPreview].option + val b = RAttachment.as("b") + val a = RAttachmentPreview.as("a") + val i = RItem.as("i") - val from = table ++ fr"a INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) - - val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) - - selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentPreview].option + Select( + select(a.all), + from(a) + .innerJoin(b, a.id === b.id) + .innerJoin(i, i.id === b.itemId), + a.id === attachId && b.id === attachId && i.cid === collective + ).build.query[RAttachmentPreview].option } def findByItem(itemId: Ident): ConnectionIO[Vector[RAttachmentPreview]] = { - val sId = Columns.id.prefix("s") - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - - val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) - selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) - .query[RAttachmentPreview] - .to[Vector] +// val sId = Columns.id.prefix("s") +// val aId = RAttachment.Columns.id.prefix("a") +// val aItem = RAttachment.Columns.itemId.prefix("a") +// +// val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) +// selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) +// .query[RAttachmentPreview] +// .to[Vector] + val s = RAttachmentPreview.as("s") + val a = RAttachment.as("a") + Select( + select(s.all), + from(s) + .innerJoin(a, s.id === a.id), + a.itemId === itemId + ).build.query[RAttachmentPreview].to[Vector] } def findByItemAndCollective( itemId: Ident, coll: Ident ): ConnectionIO[Option[RAttachmentPreview]] = { - val sId = Columns.id.prefix("s") - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val aPos = RAttachment.Columns.position.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") +// val sId = Columns.id.prefix("s") +// val aId = RAttachment.Columns.id.prefix("a") +// val aItem = RAttachment.Columns.itemId.prefix("a") +// val aPos = RAttachment.Columns.position.prefix("a") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = +// table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) +// +// selectSimple( +// all.map(_.prefix("s")) ++ List(aPos), +// from, +// and(aItem.is(itemId), iColl.is(coll)) +// ) +// .query[(RAttachmentPreview, Int)] +// .to[Vector] +// .map(_.sortBy(_._2).headOption.map(_._1)) + val s = RAttachmentPreview.as("s") + val a = RAttachment.as("a") + val i = RItem.as("i") - val from = - table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) - - selectSimple( - all.map(_.prefix("s")) ++ List(aPos), - from, - and(aItem.is(itemId), iColl.is(coll)) - ) + Select( + select(s.all).append(a.position.s), + from(s) + .innerJoin(a, s.id === a.id) + .innerJoin(i, i.id === a.itemId), + a.itemId === itemId && i.cid === coll + ).build .query[(RAttachmentPreview, Int)] .to[Vector] .map(_.sortBy(_._2).headOption.map(_._1)) @@ -102,22 +149,33 @@ object RAttachmentPreview { ): ConnectionIO[Vector[(RAttachmentPreview, FileMeta)]] = { import bitpeace.sql._ - val aId = Columns.id.prefix("a") - val afileMeta = fileId.prefix("a") - val bPos = RAttachment.Columns.position.prefix("b") - val bId = RAttachment.Columns.id.prefix("b") - val bItem = RAttachment.Columns.itemId.prefix("b") - val mId = RFileMeta.Columns.id.prefix("m") +// val aId = Columns.id.prefix("a") +// val afileMeta = fileId.prefix("a") +// val bPos = RAttachment.Columns.position.prefix("b") +// val bId = RAttachment.Columns.id.prefix("b") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val mId = RFileMeta.Columns.id.prefix("m") +// +// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) +// val from = table ++ fr"a INNER JOIN" ++ +// RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) +// val where = bItem.is(id) +// +// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) +// .query[(RAttachmentPreview, FileMeta)] +// .to[Vector] + val a = RAttachmentPreview.as("a") + val b = RAttachment.as("b") + val m = RFileMeta.as("m") - val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) - val from = table ++ fr"a INNER JOIN" ++ - RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) - val where = bItem.is(id) - - (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) - .query[(RAttachmentPreview, FileMeta)] - .to[Vector] + Select( + select(a.all, m.all), + from(a) + .innerJoin(m, a.fileId === m.id) + .innerJoin(b, b.id === a.id), + b.itemId === id + ).orderBy(b.position.asc).build.query[(RAttachmentPreview, FileMeta)].to[Vector] } } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala index f67a805f..4d94743b 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala @@ -1,8 +1,10 @@ package docspell.store.records +import cats.data.NonEmptyList + import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import bitpeace.FileMeta import doobie._ @@ -19,10 +21,24 @@ case class RAttachmentSource( ) object RAttachmentSource { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "attachment_source" + + val id = Column[Ident]("id", this) + val fileId = Column[Ident]("file_id", this) + val name = Column[String]("filename", this) + val created = Column[Timestamp]("created", this) + + val all = NonEmptyList.of[Column[_]](id, fileId, name, created) + } + + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) val table = fr"attachment_source" - object Columns { + import docspell.store.impl._ val id = Column("id") val fileId = Column("file_id") val name = Column("filename") @@ -31,67 +47,90 @@ object RAttachmentSource { val all = List(id, fileId, name, created) } - import Columns._ - def of(ra: RAttachment): RAttachmentSource = RAttachmentSource(ra.id, ra.fileId, ra.name, ra.created) def insert(v: RAttachmentSource): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run + DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] = - selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option + run(select(T.all), from(T), T.id === attachId).query[RAttachmentSource].option def isSameFile(attachId: Ident, file: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(id.is(attachId), fileId.is(file))) + Select(count(T.id).s, from(T), T.id === attachId && T.fileId === file).build .query[Int] .unique .map(_ > 0) def isConverted(attachId: Ident): ConnectionIO[Boolean] = { - val sId = Columns.id.prefix("s") - val sFile = Columns.fileId.prefix("s") - val aId = RAttachment.Columns.id.prefix("a") - val aFile = RAttachment.Columns.fileId.prefix("a") + val s = RAttachmentSource.as("s") + val a = RAttachment.as("a") + Select( + count(a.id).s, + from(s).innerJoin(a, a.id === s.id), + a.id === attachId && a.fileId <> s.fileId + ).build.query[Int].unique.map(_ > 0) - val from = table ++ fr"s INNER JOIN" ++ - RAttachment.table ++ fr"a ON" ++ aId.is(sId) - - selectCount(aId, from, and(aId.is(attachId), aFile.isNot(sFile))) - .query[Int] - .unique - .map(_ > 0) +// val sId = Columns.id.prefix("s") +// val sFile = Columns.fileId.prefix("s") +// val aId = RAttachment.Columns.id.prefix("a") +// val aFile = RAttachment.Columns.fileId.prefix("a") +// +// val from = table ++ fr"s INNER JOIN" ++ +// RAttachment.table ++ fr"a ON" ++ aId.is(sId) +// +// selectCount(aId, from, and(aId.is(attachId), aFile.isNot(sFile))) +// .query[Int] +// .unique +// .map(_ > 0) } def delete(attachId: Ident): ConnectionIO[Int] = - deleteFrom(table, id.is(attachId)).update.run + DML.delete(T, T.id === attachId) def findByIdAndCollective( attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentSource]] = { - val bId = RAttachment.Columns.id.prefix("b") - val aId = Columns.id.prefix("a") - val bItem = RAttachment.Columns.itemId.prefix("b") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") +// val bId = RAttachment.Columns.id.prefix("b") +// val aId = Columns.id.prefix("a") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val iId = RItem.Columns.id.prefix("i") +// val iColl = RItem.Columns.cid.prefix("i") +// +// val from = table ++ fr"a INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ +// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) +// +// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) +// +// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentSource].option + val b = RAttachment.as("b") + val a = RAttachmentSource.as("a") + val i = RItem.as("i") - val from = table ++ fr"a INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) - - val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) - - selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentSource].option + Select( + select(a.all), + from(a) + .innerJoin(b, a.id === b.id) + .innerJoin(i, i.id === b.itemId), + a.id === attachId && b.id === attachId && i.cid === collective + ).build.query[RAttachmentSource].option } def findByItem(itemId: Ident): ConnectionIO[Vector[RAttachmentSource]] = { - val sId = Columns.id.prefix("s") - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") +// val sId = Columns.id.prefix("s") +// val aId = RAttachment.Columns.id.prefix("a") +// val aItem = RAttachment.Columns.itemId.prefix("a") +// +// val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) +// selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) +// .query[RAttachmentSource] +// .to[Vector] - val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) - selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) + val s = RAttachmentSource.as("s") + val a = RAttachment.as("a") + Select(select(s.all), from(s).innerJoin(a, a.id === s.id), a.itemId === itemId).build .query[RAttachmentSource] .to[Vector] } @@ -101,22 +140,33 @@ object RAttachmentSource { ): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = { import bitpeace.sql._ - val aId = Columns.id.prefix("a") - val afileMeta = fileId.prefix("a") - val bPos = RAttachment.Columns.position.prefix("b") - val bId = RAttachment.Columns.id.prefix("b") - val bItem = RAttachment.Columns.itemId.prefix("b") - val mId = RFileMeta.Columns.id.prefix("m") +// val aId = Columns.id.prefix("a") +// val afileMeta = fileId.prefix("a") +// val bPos = RAttachment.Columns.position.prefix("b") +// val bId = RAttachment.Columns.id.prefix("b") +// val bItem = RAttachment.Columns.itemId.prefix("b") +// val mId = RFileMeta.Columns.id.prefix("m") +// +// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) +// val from = table ++ fr"a INNER JOIN" ++ +// RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ +// RAttachment.table ++ fr"b ON" ++ aId.is(bId) +// val where = bItem.is(id) +// +// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) +// .query[(RAttachmentSource, FileMeta)] +// .to[Vector] + val a = RAttachmentSource.as("a") + val b = RAttachment.as("b") + val m = RFileMeta.as("m") - val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) - val from = table ++ fr"a INNER JOIN" ++ - RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ - RAttachment.table ++ fr"b ON" ++ aId.is(bId) - val where = bItem.is(id) - - (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) - .query[(RAttachmentSource, FileMeta)] - .to[Vector] + Select( + select(a.all, m.all), + from(a) + .innerJoin(m, a.fileId === m.id) + .innerJoin(b, b.id === a.id), + b.itemId === id + ).orderBy(b.position.asc).build.query[(RAttachmentSource, FileMeta)].to[Vector] } } diff --git a/modules/store/src/main/scala/docspell/store/records/RClassifierSetting.scala b/modules/store/src/main/scala/docspell/store/records/RClassifierSetting.scala index 680741a0..749435d1 100644 --- a/modules/store/src/main/scala/docspell/store/records/RClassifierSetting.scala +++ b/modules/store/src/main/scala/docspell/store/records/RClassifierSetting.scala @@ -1,10 +1,11 @@ package docspell.store.records +import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import com.github.eikek.calev._ import doobie._ @@ -21,71 +22,69 @@ case class RClassifierSetting( ) {} object RClassifierSetting { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "classifier_setting" - val table = fr"classifier_setting" - - object Columns { - val cid = Column("cid") - val enabled = Column("enabled") - val schedule = Column("schedule") - val category = Column("category") - val itemCount = Column("item_count") - val fileId = Column("file_id") - val created = Column("created") - val all = List(cid, enabled, schedule, category, itemCount, fileId, created) - } - import Columns._ - - def insert(v: RClassifierSetting): ConnectionIO[Int] = { - val sql = - insertRow( - table, - all, - fr"${v.cid},${v.enabled},${v.schedule},${v.category},${v.itemCount},${v.fileId},${v.created}" - ) - sql.update.run + val cid = Column[Ident]("cid", this) + val enabled = Column[Boolean]("enabled", this) + val schedule = Column[CalEvent]("schedule", this) + val category = Column[String]("category", this) + val itemCount = Column[Int]("item_count", this) + val fileId = Column[Ident]("file_id", this) + val created = Column[Timestamp]("created", this) + val all = NonEmptyList + .of[Column[_]](cid, enabled, schedule, category, itemCount, fileId, created) } - def updateAll(v: RClassifierSetting): ConnectionIO[Int] = { - val sql = updateRow( - table, - cid.is(v.cid), - commas( - enabled.setTo(v.enabled), - schedule.setTo(v.schedule), - category.setTo(v.category), - itemCount.setTo(v.itemCount), - fileId.setTo(v.fileId) + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + + def insert(v: RClassifierSetting): ConnectionIO[Int] = + DML.insert( + T, + T.all, + fr"${v.cid},${v.enabled},${v.schedule},${v.category},${v.itemCount},${v.fileId},${v.created}" + ) + + def updateAll(v: RClassifierSetting): ConnectionIO[Int] = + DML.update( + T, + T.cid === v.cid, + DML.set( + T.enabled.setTo(v.enabled), + T.schedule.setTo(v.schedule), + T.category.setTo(v.category), + T.itemCount.setTo(v.itemCount), + T.fileId.setTo(v.fileId) ) ) - sql.update.run - } def updateFile(coll: Ident, fid: Ident): ConnectionIO[Int] = - updateRow(table, cid.is(coll), fileId.setTo(fid)).update.run + DML.update(T, T.cid === coll, DML.set(T.fileId.setTo(fid))) def updateSettings(v: RClassifierSetting): ConnectionIO[Int] = for { - n1 <- updateRow( - table, - cid.is(v.cid), - commas( - enabled.setTo(v.enabled), - schedule.setTo(v.schedule), - itemCount.setTo(v.itemCount), - category.setTo(v.category) + n1 <- DML.update( + T, + T.cid === v.cid, + DML.set( + T.enabled.setTo(v.enabled), + T.schedule.setTo(v.schedule), + T.itemCount.setTo(v.itemCount), + T.category.setTo(v.category) ) - ).update.run + ) n2 <- if (n1 <= 0) insert(v) else 0.pure[ConnectionIO] } yield n1 + n2 def findById(id: Ident): ConnectionIO[Option[RClassifierSetting]] = { - val sql = selectSimple(all, table, cid.is(id)) + val sql = run(select(T.all), from(T), T.cid === id) sql.query[RClassifierSetting].option } def delete(coll: Ident): ConnectionIO[Int] = - deleteFrom(table, cid.is(coll)).update.run + DML.delete(T, T.cid === coll) case class Classifier( enabled: Boolean, diff --git a/modules/store/src/main/scala/docspell/store/records/RCollective.scala b/modules/store/src/main/scala/docspell/store/records/RCollective.scala index 2487ed22..ca3b2666 100644 --- a/modules/store/src/main/scala/docspell/store/records/RCollective.scala +++ b/modules/store/src/main/scala/docspell/store/records/RCollective.scala @@ -1,10 +1,11 @@ package docspell.store.records +import cats.data.NonEmptyList import fs2.Stream import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -18,58 +19,54 @@ case class RCollective( ) object RCollective { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "collective" - val table = fr"collective" + val id = Column[Ident]("cid", this) + val state = Column[CollectiveState]("state", this) + val language = Column[Language]("doclang", this) + val integration = Column[Boolean]("integration_enabled", this) + val created = Column[Timestamp]("created", this) - object Columns { - - val id = Column("cid") - val state = Column("state") - val language = Column("doclang") - val integration = Column("integration_enabled") - val created = Column("created") - - val all = List(id, state, language, integration, created) + val all = NonEmptyList.of[Column[_]](id, state, language, integration, created) } - import Columns._ + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) - def insert(value: RCollective): ConnectionIO[Int] = { - val sql = insertRow( - table, - Columns.all, + def insert(value: RCollective): ConnectionIO[Int] = + DML.insert( + T, + T.all, fr"${value.id},${value.state},${value.language},${value.integrationEnabled},${value.created}" ) - sql.update.run - } - def update(value: RCollective): ConnectionIO[Int] = { - val sql = updateRow( - table, - id.is(value.id), - commas( - state.setTo(value.state) + def update(value: RCollective): ConnectionIO[Int] = + DML.update( + T, + T.id === value.id, + DML.set( + T.state.setTo(value.state) ) ) - sql.update.run - } def findLanguage(cid: Ident): ConnectionIO[Option[Language]] = - selectSimple(List(language), table, id.is(cid)).query[Option[Language]].unique + Select(T.language.s, from(T), T.id === cid).build.query[Option[Language]].unique def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] = - updateRow(table, id.is(cid), language.setTo(lang)).update.run + DML.update(T, T.id === cid, DML.set(T.language.setTo(lang))) def updateSettings(cid: Ident, settings: Settings): ConnectionIO[Int] = for { - n1 <- updateRow( - table, - id.is(cid), - commas( - language.setTo(settings.language), - integration.setTo(settings.integrationEnabled) + n1 <- DML.update( + T, + T.id === cid, + DML.set( + T.language.setTo(settings.language), + T.integration.setTo(settings.integrationEnabled) ) - ).update.run + ) cls <- Timestamp .current[ConnectionIO] @@ -83,66 +80,64 @@ object RCollective { } yield n1 + n2 def getSettings(coll: Ident): ConnectionIO[Option[Settings]] = { - val cId = id.prefix("c") - val CS = RClassifierSetting.Columns - val csCid = CS.cid.prefix("cs") + val c = RCollective.as("c") + val cs = RClassifierSetting.as("cs") - val cols = Seq( - language.prefix("c"), - integration.prefix("c"), - CS.enabled.prefix("cs"), - CS.schedule.prefix("cs"), - CS.itemCount.prefix("cs"), - CS.category.prefix("cs") - ) - val from = table ++ fr"c LEFT JOIN" ++ - RClassifierSetting.table ++ fr"cs ON" ++ csCid.is(cId) - - selectSimple(cols, from, cId.is(coll)) - .query[Settings] - .option + Select( + select( + c.language.s, + c.integration.s, + cs.enabled.s, + cs.schedule.s, + cs.itemCount.s, + cs.category.s + ), + from(c).leftJoin(cs, cs.cid === c.id), + c.id === coll + ).build.query[Settings].option } def findById(cid: Ident): ConnectionIO[Option[RCollective]] = { - val sql = selectSimple(all, table, id.is(cid)) + val sql = run(select(T.all), from(T), T.id === cid) sql.query[RCollective].option } def findByItem(itemId: Ident): ConnectionIO[Option[RCollective]] = { - val iColl = RItem.Columns.cid.prefix("i") - val iId = RItem.Columns.id.prefix("i") - val cId = id.prefix("c") - val from = RItem.table ++ fr"i INNER JOIN" ++ table ++ fr"c ON" ++ iColl.is(cId) - selectSimple(all.map(_.prefix("c")), from, iId.is(itemId)).query[RCollective].option + val i = RItem.as("i") + val c = RCollective.as("c") + Select( + select(c.all), + from(i).innerJoin(c, i.cid === c.id), + i.id === itemId + ).build.query[RCollective].option } def existsById(cid: Ident): ConnectionIO[Boolean] = { - val sql = selectCount(id, table, id.is(cid)) + val sql = Select(count(T.id).s, from(T), T.id === cid).build sql.query[Int].unique.map(_ > 0) } - def findAll(order: Columns.type => Column): ConnectionIO[Vector[RCollective]] = { - val sql = selectSimple(all, table, Fragment.empty) ++ orderBy(order(Columns).f) - sql.query[RCollective].to[Vector] + def findAll(order: Table => Column[_]): ConnectionIO[Vector[RCollective]] = { + val sql = Select(select(T.all), from(T)).orderBy(order(T)) + sql.build.query[RCollective].to[Vector] } - def streamAll(order: Columns.type => Column): Stream[ConnectionIO, RCollective] = { - val sql = selectSimple(all, table, Fragment.empty) ++ orderBy(order(Columns).f) - sql.query[RCollective].stream + def streamAll(order: Table => Column[_]): Stream[ConnectionIO, RCollective] = { + val sql = Select(select(T.all), from(T)).orderBy(order(T)) + sql.build.query[RCollective].stream } def findByAttachment(attachId: Ident): ConnectionIO[Option[RCollective]] = { - val iColl = RItem.Columns.cid.prefix("i") - val iId = RItem.Columns.id.prefix("i") - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val cId = Columns.id.prefix("c") - - val from = table ++ fr"c INNER JOIN" ++ - RItem.table ++ fr"i ON" ++ cId.is(iColl) ++ fr"INNER JOIN" ++ - RAttachment.table ++ fr"a ON" ++ aItem.is(iId) - - selectSimple(all.map(_.prefix("c")), from, aId.is(attachId)).query[RCollective].option + val i = RItem.as("i") + val a = RAttachment.as("a") + val c = RCollective.as("c") + Select( + select(c.all), + from(c) + .innerJoin(i, c.id === i.cid) + .innerJoin(a, a.itemId === i.id), + a.id === attachId + ).build.query[RCollective].option } case class Settings( diff --git a/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala b/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala index b9e73f77..2567562f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala @@ -1,11 +1,13 @@ package docspell.store.records +import java.time.Instant + import cats.data.NonEmptyList import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.syntax.MimeTypes._ import bitpeace.FileMeta @@ -14,26 +16,30 @@ import doobie._ import doobie.implicits._ object RFileMeta { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "filemeta" - val table = fr"filemeta" + val id = Column[Ident]("id", this) + val timestamp = Column[Instant]("timestamp", this) + val mimetype = Column[Mimetype]("mimetype", this) + val length = Column[Long]("length", this) + val checksum = Column[String]("checksum", this) + val chunks = Column[Int]("chunks", this) + val chunksize = Column[Int]("chunksize", this) - object Columns { - val id = Column("id") - val timestamp = Column("timestamp") - val mimetype = Column("mimetype") - val length = Column("length") - val checksum = Column("checksum") - val chunks = Column("chunks") - val chunksize = Column("chunksize") - - val all = List(id, timestamp, mimetype, length, checksum, chunks, chunksize) + val all = NonEmptyList + .of[Column[_]](id, timestamp, mimetype, length, checksum, chunks, chunksize) } + val T = Table(None) + def as(alias: String): Table = + Table(Some(alias)) + def findById(fid: Ident): ConnectionIO[Option[FileMeta]] = { import bitpeace.sql._ - selectSimple(Columns.all, table, Columns.id.is(fid)).query[FileMeta].option + run(select(T.all), from(T), T.id === fid).query[FileMeta].option } def findByIds(ids: List[Ident]): ConnectionIO[Vector[FileMeta]] = { @@ -41,7 +47,7 @@ object RFileMeta { NonEmptyList.fromList(ids) match { case Some(nel) => - selectSimple(Columns.all, table, Columns.id.isIn(nel)).query[FileMeta].to[Vector] + run(select(T.all), from(T), T.id.in(nel)).query[FileMeta].to[Vector] case None => Vector.empty[FileMeta].pure[ConnectionIO] } @@ -50,7 +56,7 @@ object RFileMeta { def findMime(fid: Ident): ConnectionIO[Option[MimeType]] = { import bitpeace.sql._ - selectSimple(Seq(Columns.mimetype), table, Columns.id.is(fid)) + run(select(T.mimetype), from(T), T.id === fid) .query[Mimetype] .option .map(_.map(_.toLocal)) diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index eba4db3a..929b3528 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -5,9 +5,8 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.store.impl.Implicits._ -import docspell.store.impl._ -import docspell.store.qb.{Select, TableDef} +import docspell.store.qb.DSL._ +import docspell.store.qb._ import doobie._ import doobie.implicits._ @@ -110,8 +109,9 @@ object RItem { Table(Some(alias)) val table = fr"item" - object Columns { + import docspell.store.impl._ + val id = Column("itemid") val cid = Column("cid") val name = Column("name") @@ -149,19 +149,21 @@ object RItem { folder ) } - import Columns._ + + private val currentTime = + Timestamp.current[ConnectionIO] def insert(v: RItem): ConnectionIO[Int] = - insertRow( - table, - all, + DML.insert( + T, + T.all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++ fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++ fr"${v.created},${v.updated},${v.notes},${v.folderId}" - ).update.run + ) def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(cid), table, id.is(itemId)).query[Ident].option + Select(T.cid.s, from(T), T.id === itemId).build.query[Ident].option def updateState( itemId: Ident, @@ -170,11 +172,11 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.is(itemId), state.isIn(existing)), - commas(state.setTo(itemState), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id === itemId && T.state.in(existing), + DML.set(T.state.setTo(itemState), T.updated.setTo(t)) + ) } yield n def updateStateForCollective( @@ -184,11 +186,11 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(state.setTo(itemState), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.state.setTo(itemState), T.updated.setTo(t)) + ) } yield n def updateDirection( @@ -198,11 +200,11 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(incoming.setTo(dir), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.incoming.setTo(dir), T.updated.setTo(t)) + ) } yield n def updateCorrOrg( @@ -212,21 +214,21 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(corrOrg.setTo(org), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.corrOrg.setTo(org), T.updated.setTo(t)) + ) } yield n def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(cid.is(coll), corrOrg.is(Some(currentOrg))), - commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.cid === coll && T.corrOrg === currentOrg, + DML.set(T.corrOrg.setTo(None: Option[Ident]), T.updated.setTo(t)) + ) } yield n def updateCorrPerson( @@ -236,21 +238,21 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(corrPerson.setTo(person), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.corrPerson.setTo(person), T.updated.setTo(t)) + ) } yield n def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(cid.is(coll), corrPerson.is(Some(currentPerson))), - commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.cid === coll && T.corrPerson === currentPerson, + DML.set(T.corrPerson.setTo(None: Option[Ident]), T.updated.setTo(t)) + ) } yield n def updateConcPerson( @@ -260,21 +262,21 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(concPerson.setTo(person), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.concPerson.setTo(person), T.updated.setTo(t)) + ) } yield n def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(cid.is(coll), concPerson.is(Some(currentPerson))), - commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.cid === coll && T.concPerson === currentPerson, + DML.set(T.concPerson.setTo(None: Option[Ident]), T.updated.setTo(t)) + ) } yield n def updateConcEquip( @@ -284,21 +286,21 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(concEquipment.setTo(equip), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.concEquipment.setTo(equip), T.updated.setTo(t)) + ) } yield n def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(cid.is(coll), concEquipment.is(Some(currentEquip))), - commas(concEquipment.setTo(None: Option[Ident]), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.cid === coll && T.concEquipment === currentEquip, + DML.set(T.concEquipment.setTo(None: Option[Ident]), T.updated.setTo(t)) + ) } yield n def updateFolder( @@ -308,31 +310,31 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(cid.is(coll), id.is(itemId)), - commas(folder.setTo(folderId), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.cid === coll && T.id === itemId, + DML.set(T.folder.setTo(folderId), T.updated.setTo(t)) + ) } yield n def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.is(itemId), cid.is(coll)), - commas(notes.setTo(text), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id === itemId && T.cid === coll, + DML.set(T.notes.setTo(text), T.updated.setTo(t)) + ) } yield n def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.is(itemId), cid.is(coll)), - commas(name.setTo(itemName), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id === itemId && T.cid === coll, + DML.set(T.name.setTo(itemName), T.updated.setTo(t)) + ) } yield n def updateDate( @@ -342,11 +344,11 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(itemDate.setTo(date), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.itemDate.setTo(date), T.updated.setTo(t)) + ) } yield n def updateDueDate( @@ -356,50 +358,50 @@ object RItem { ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow( - table, - and(id.isIn(itemIds), cid.is(coll)), - commas(dueDate.setTo(date), updated.setTo(t)) - ).update.run + n <- DML.update( + T, + T.id.in(itemIds) && T.cid === coll, + DML.set(T.dueDate.setTo(date), T.updated.setTo(t)) + ) } yield n def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(id.is(itemId), cid.is(coll))).update.run + DML.delete(T, T.id === itemId && T.cid === coll) def existsById(itemId: Ident): ConnectionIO[Boolean] = - selectCount(id, table, id.is(itemId)).query[Int].unique.map(_ > 0) + Select(count(T.id).s, from(T), T.id === itemId).build.query[Int].unique.map(_ > 0) def existsByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Boolean] = - selectCount(id, table, and(id.is(itemId), cid.is(coll))).query[Int].unique.map(_ > 0) + Select(count(T.id).s, from(T), T.id === itemId && T.cid === coll).build + .query[Int] + .unique + .map(_ > 0) def existsByIdsAndCollective( itemIds: NonEmptyList[Ident], coll: Ident ): ConnectionIO[Boolean] = - selectCount(id, table, and(id.isIn(itemIds), cid.is(coll))) + Select(count(T.id).s, from(T), T.id.in(itemIds) && T.cid === coll).build .query[Int] .unique .map(_ == itemIds.size) def findByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Option[RItem]] = - selectSimple(all, table, and(id.is(itemId), cid.is(coll))).query[RItem].option + run(select(T.all), from(T), T.id === itemId && T.cid === coll).query[RItem].option def findById(itemId: Ident): ConnectionIO[Option[RItem]] = - selectSimple(all, table, id.is(itemId)).query[RItem].option + run(select(T.all), from(T), T.id === itemId).query[RItem].option def checkByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Option[Ident]] = - selectSimple(Seq(id), table, and(id.is(itemId), cid.is(coll))).query[Ident].option + Select(T.id.s, from(T), T.id === itemId && T.cid === coll).build.query[Ident].option def removeFolder(folderId: Ident): ConnectionIO[Int] = { val empty: Option[Ident] = None - updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run + DML.update(T, T.folder === folderId, DML.set(T.folder.setTo(empty))) } - def filterItemsFragment(items: NonEmptyList[Ident], coll: Ident): Select = { - import docspell.store.qb.DSL._ - + def filterItemsFragment(items: NonEmptyList[Ident], coll: Ident): Select = Select(select(T.id), from(T), T.cid === coll && T.id.in(items)) - } def filterItems(items: NonEmptyList[Ident], coll: Ident): ConnectionIO[Vector[Ident]] = filterItemsFragment(items, coll).build.query[Ident].to[Vector] From a355767fdbdcd8300a1d32448e11dadb1f3883bb Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 21:15:53 +0100 Subject: [PATCH 15/38] Convert all query libs besides QItem --- .../docspell/backend/ops/OItemSearch.scala | 4 +- .../main/scala/docspell/store/qb/Select.scala | 5 ++ .../scala/docspell/store/queries/Batch.scala | 22 +++++ .../docspell/store/queries/QAttachment.scala | 90 +++++++------------ .../docspell/store/queries/QCollective.scala | 42 ++++----- .../docspell/store/queries/QCustomField.scala | 35 +------- .../scala/docspell/store/queries/QItem.scala | 21 ----- .../scala/docspell/store/queries/QMails.scala | 72 ++++++--------- .../store/queries/QOrganization.scala | 44 ++++----- 9 files changed, 122 insertions(+), 213 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/Batch.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index c546a184..25efda8d 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -59,8 +59,8 @@ object OItemSearch { type Query = QItem.Query val Query = QItem.Query - type Batch = QItem.Batch - val Batch = QItem.Batch + type Batch = docspell.store.queries.Batch + val Batch = docspell.store.queries.Batch type ListItem = QItem.ListItem val ListItem = QItem.ListItem diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index 406135f0..e219ee03 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -64,6 +64,11 @@ object Select { def distinct: SimpleSelect = copy(distinctFlag = true) + + def where(c: Option[Condition]): SimpleSelect = + copy(where = c) + def where(c: Condition): SimpleSelect = + copy(where = Some(c)) } case class Union(q: Select, qs: Vector[Select]) extends Select diff --git a/modules/store/src/main/scala/docspell/store/queries/Batch.scala b/modules/store/src/main/scala/docspell/store/queries/Batch.scala new file mode 100644 index 00000000..d88ec957 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/Batch.scala @@ -0,0 +1,22 @@ +package docspell.store.queries + +case class Batch(offset: Int, limit: Int) { + def restrictLimitTo(n: Int): Batch = + Batch(offset, math.min(n, limit)) + + def next: Batch = + Batch(offset + limit, limit) + + def first: Batch = + Batch(0, limit) +} + +object Batch { + val all: Batch = Batch(0, Int.MaxValue) + + def page(n: Int, size: Int): Batch = + Batch(n * size, size) + + def limit(c: Int): Batch = + Batch(0, c) +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index f1aae89a..6ac9327a 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -8,15 +8,20 @@ import fs2.Stream import docspell.common._ import docspell.common.syntax.all._ import docspell.store.Store -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ -import doobie.implicits._ object QAttachment { private[this] val logger = org.log4s.getLogger + private val a = RAttachment.as("a") + private val item = RItem.as("i") + private val am = RAttachmentMeta.as("am") + private val c = RCollective.as("c") + def deletePreview[F[_]: Sync](store: Store[F])(attachId: Ident): F[Int] = { val findPreview = for { @@ -113,20 +118,13 @@ object QAttachment { } yield ns.sum def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = { - val AC = RAttachment.Columns - val MC = RAttachmentMeta.Columns - val IC = RItem.Columns - - val q = fr"SELECT" ++ MC.proposals - .prefix("m") - .f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ - fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id - .prefix("a") - .is(MC.id.prefix("m")) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId - .prefix("a") - .is(IC.id.prefix("i")) ++ - fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll)) + val q = Select( + am.proposals.s, + from(am) + .innerJoin(a, a.id === am.id) + .innerJoin(item, a.itemId === item.id), + a.itemId === itemId && item.cid === coll + ).build for { ml <- q.query[MetaProposalList].to[Vector] @@ -137,24 +135,13 @@ object QAttachment { attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentMeta]] = { - val AC = RAttachment.Columns - val MC = RAttachmentMeta.Columns - val IC = RItem.Columns - - val q = - fr"SELECT" ++ commas( - MC.all.map(_.prefix("m").f) - ) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ - fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id - .prefix("i") - .is(AC.itemId.prefix("a")) ++ - fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id - .prefix("a") - .is(MC.id.prefix("m")) ++ - fr"WHERE" ++ and( - AC.id.prefix("a").is(attachId), - IC.cid.prefix("i").is(collective) - ) + val q = Select( + select(am.all), + from(item) + .innerJoin(a, a.itemId === item.id) + .innerJoin(am, am.id === a.id), + a.id === attachId && item.cid === collective + ).build q.query[RAttachmentMeta].option } @@ -171,31 +158,16 @@ object QAttachment { def allAttachmentMetaAndName( coll: Option[Ident], chunkSize: Int - ): Stream[ConnectionIO, ContentAndName] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val aName = RAttachment.Columns.name.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mContent = RAttachmentMeta.Columns.content.prefix("m") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val iFolder = RItem.Columns.folder.prefix("i") - val c = RCollective.as("c") - val cId = c.id.column - val cLang = c.language.column - - val cols = Seq(aId, aItem, iColl, iFolder, cLang, aName, mContent) - val from = RAttachment.table ++ fr"a INNER JOIN" ++ - RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ - fr"INNER JOIN" ++ Fragment.const(RCollective.T.tableName) ++ fr"c ON" ++ cId.is( - iColl - ) - - val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty) - - selectSimple(cols, from, where) + ): Stream[ConnectionIO, ContentAndName] = + Select( + select(a.id, a.itemId, item.cid, item.folder, c.language, a.name, am.content), + from(a) + .innerJoin(am, am.id === a.id) + .innerJoin(item, item.id === a.itemId) + .innerJoin(c, c.id === item.cid) + ).where(coll.map(cid => item.cid === cid)) + .build .query[ContentAndName] .streamWithChunkSize(chunkSize) - } + } diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index 36361780..b9e8f74a 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -5,14 +5,20 @@ import fs2.Stream import docspell.common.ContactKind import docspell.common.{Direction, Ident} -import docspell.store.impl.Implicits._ -import docspell.store.qb.{GroupBy, Select} +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ import doobie.implicits._ object QCollective { + private val ti = RTagItem.as("ti") + private val t = RTag.as("t") + private val ro = ROrganization.as("o") + private val rp = RPerson.as("p") + private val rc = RContact.as("c") + private val i = RItem.as("i") case class Names(org: Vector[String], pers: Vector[String], equip: Vector[String]) object Names { @@ -37,17 +43,16 @@ object QCollective { ) def getInsights(coll: Ident): ConnectionIO[InsightData] = { - val IC = RItem.Columns - val q0 = selectCount( - IC.id, - RItem.table, - and(IC.cid.is(coll), IC.incoming.is(Direction.incoming)) - ).query[Int].unique - val q1 = selectCount( - IC.id, - RItem.table, - and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing)) - ).query[Int].unique + val q0 = Select( + count(i.id).s, + from(i), + i.cid === coll && i.incoming === Direction.incoming + ).build.query[Int].unique + val q1 = Select( + count(i.id).s, + from(i), + i.cid === coll && i.incoming === Direction.outgoing + ).build.query[Int].unique val fileSize = sql""" select sum(length) from ( @@ -78,10 +83,6 @@ object QCollective { } def tagCloud(coll: Ident): ConnectionIO[List[TagCount]] = { - import docspell.store.qb.DSL._ - - val ti = RTagItem.as("ti") - val t = RTag.as("t") val sql = Select( select(t.all).append(count(ti.itemId).s), @@ -97,13 +98,6 @@ object QCollective { query: Option[String], kind: Option[ContactKind] ): Stream[ConnectionIO, RContact] = { - import docspell.store.qb.DSL._ - import docspell.store.qb._ - - val ro = ROrganization.as("o") - val rp = RPerson.as("p") - val rc = RContact.as("c") - val orgCond = Select(select(ro.oid), from(ro), ro.cid === coll) val persCond = Select(select(rp.pid), from(rp), rp.cid === coll) val valueFilter = query.map(s => rc.value.like(s"%${s.toLowerCase}%")) diff --git a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala index 0990a12b..b2923295 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala @@ -29,45 +29,14 @@ object QCustomField { nameQuery: Option[String], fieldId: Option[Ident] ): Select = { -// val fId = RCustomField.Columns.id.prefix("f") -// val fColl = RCustomField.Columns.cid.prefix("f") -// val fName = RCustomField.Columns.name.prefix("f") -// val fLabel = RCustomField.Columns.label.prefix("f") -// val vField = RCustomFieldValue.Columns.field.prefix("v") -// -// val join = RCustomField.table ++ fr"f LEFT OUTER JOIN" ++ -// RCustomFieldValue.table ++ fr"v ON" ++ fId.is(vField) -// -// val cols = RCustomField.Columns.all.map(_.prefix("f")) :+ Column("COUNT(v.id)") -// -// val nameCond = nameQuery.map(QueryWildcard.apply) match { -// case Some(q) => -// or(fName.lowerLike(q), and(fLabel.isNotNull, fLabel.lowerLike(q))) -// case None => -// Fragment.empty -// } -// val fieldCond = fieldId match { -// case Some(id) => -// fId.is(id) -// case None => -// Fragment.empty -// } -// val cond = and(fColl.is(coll), nameCond, fieldCond) -// -// val group = NonEmptyList.fromList(RCustomField.Columns.all) match { -// case Some(nel) => groupBy(nel.map(_.prefix("f"))) -// case None => Fragment.empty -// } -// -// selectSimple(cols, join, cond) ++ group - val nameFilter = nameQuery.map { q => f.name.likes(q) || (f.label.isNotNull && f.label.like(q)) } Select( f.all.map(_.s).append(count(v.id).as("num")), - from(f).leftJoin(v, f.id === v.field), + from(f) + .leftJoin(v, f.id === v.field), f.cid === coll &&? nameFilter &&? fieldId.map(fid => f.id === fid), GroupBy(f.all) ) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 81bb78b0..5c9ca443 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -259,27 +259,6 @@ object QItem { ) } - case class Batch(offset: Int, limit: Int) { - def restrictLimitTo(n: Int): Batch = - Batch(offset, math.min(n, limit)) - - def next: Batch = - Batch(offset + limit, limit) - - def first: Batch = - Batch(0, limit) - } - - object Batch { - val all: Batch = Batch(0, Int.MaxValue) - - def page(n: Int, size: Int): Batch = - Batch(n * size, size) - - def limit(c: Int): Batch = - Batch(0, c) - } - private def findCustomFieldValuesForColl( coll: Ident, values: Seq[CustomValue] diff --git a/modules/store/src/main/scala/docspell/store/queries/QMails.scala b/modules/store/src/main/scala/docspell/store/queries/QMails.scala index 30d476af..b58740c7 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QMails.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QMails.scala @@ -3,8 +3,8 @@ package docspell.store.queries import cats.data.OptionT import docspell.common._ -import docspell.store.impl.Column -import docspell.store.impl.Implicits._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ import docspell.store.records._ import doobie._ @@ -12,6 +12,11 @@ import doobie.implicits._ object QMails { + private val item = RItem.as("i") + private val smail = RSentMail.as("sm") + private val mailitem = RSentMailItem.as("mi") + private val user = RUser.as("u") + def delete(coll: Ident, mailId: Ident): ConnectionIO[Int] = (for { m <- OptionT(findMail(coll, mailId)) @@ -19,53 +24,28 @@ object QMails { n <- OptionT.liftF(RSentMail.delete(m._1.id)) } yield k + n).getOrElse(0) - def findMail(coll: Ident, mailId: Ident): ConnectionIO[Option[(RSentMail, Ident)]] = { - val iColl = RItem.Columns.cid.prefix("i") - val smail = RSentMail.as("m") - val mId = smail.id.column + def findMail(coll: Ident, mailId: Ident): ConnectionIO[Option[(RSentMail, Ident)]] = + partialFind + .where(smail.id === mailId && item.cid === coll) + .build + .query[(RSentMail, Ident)] + .option - val (cols, from) = partialFind - - val cond = Seq(mId.is(mailId), iColl.is(coll)) - - selectSimple(cols, from, and(cond)).query[(RSentMail, Ident)].option - } - - def findMails(coll: Ident, itemId: Ident): ConnectionIO[Vector[(RSentMail, Ident)]] = { - val smailitem = RSentMailItem.as("t") - val smail = RSentMail.as("m") - val iColl = RItem.Columns.cid.prefix("i") - val tItem = smailitem.itemId.column - val mCreated = smail.created.column - - val (cols, from) = partialFind - - val cond = Seq(tItem.is(itemId), iColl.is(coll)) - - (selectSimple(cols, from, and(cond)) ++ orderBy(mCreated.f) ++ fr"DESC") + def findMails(coll: Ident, itemId: Ident): ConnectionIO[Vector[(RSentMail, Ident)]] = + partialFind + .where(mailitem.itemId === itemId && item.cid === coll) + .orderBy(smail.created.desc) + .build .query[(RSentMail, Ident)] .to[Vector] - } - private def partialFind: (Seq[Column], Fragment) = { - val user = RUser.as("u") - val smailitem = RSentMailItem.as("t") - val smail = RSentMail.as("m") - val iId = RItem.Columns.id.prefix("i") - val tItem = smailitem.itemId.column - val tMail = smailitem.sentMailId.column - val mId = smail.id.column - val mUser = smail.uid.column - - val cols = smail.all.map(_.column) :+ user.login.column - val from = Fragment.const(smail.tableName) ++ fr"m INNER JOIN" ++ - Fragment.const(smailitem.tableName) ++ fr"t ON" ++ tMail.is(mId) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ tItem.is(iId) ++ - fr"INNER JOIN" ++ Fragment.const(user.tableName) ++ fr"u ON" ++ user.uid.column.is( - mUser - ) - - (cols.toList, from) - } + private def partialFind: Select.SimpleSelect = + Select( + select(smail.all).append(user.login.s), + from(smail) + .innerJoin(mailitem, mailitem.sentMailId === smail.id) + .innerJoin(item, mailitem.itemId === item.id) + .innerJoin(user, user.uid === smail.uid) + ) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala index 585d2fd0..d83e2a25 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala @@ -13,15 +13,15 @@ import doobie._ import doobie.implicits._ object QOrganization { + private val p = RPerson.as("p") + private val c = RContact.as("c") + private val org = ROrganization.as("o") def findOrgAndContact( coll: Ident, query: Option[String], order: ROrganization.Table => Column[_] ): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = { - val org = ROrganization.as("o") - val c = RContact.as("c") - val valFilter = query.map { q => val v = s"%$q%" c.value.like(v) || org.name.like(v) || org.notes.like(v) @@ -46,9 +46,6 @@ object QOrganization { coll: Ident, orgId: Ident ): ConnectionIO[Option[(ROrganization, Vector[RContact])]] = { - val org = ROrganization.as("o") - val c = RContact.as("c") - val sql = run( select(org.all, c.all), from(org).leftJoin(c, c.orgId === org.oid), @@ -72,19 +69,16 @@ object QOrganization { query: Option[String], order: RPerson.Table => Column[_] ): Stream[ConnectionIO, (RPerson, Option[ROrganization], Vector[RContact])] = { - val pers = RPerson.as("p") - val org = ROrganization.as("o") - val c = RContact.as("c") val valFilter = query .map(s => s"%$s%") - .map(v => c.value.like(v) || pers.name.like(v) || pers.notes.like(v)) + .map(v => c.value.like(v) || p.name.like(v) || p.notes.like(v)) val sql = Select( - select(pers.all, org.all, c.all), - from(pers) - .leftJoin(org, org.oid === pers.oid) - .leftJoin(c, c.personId === pers.pid), - pers.cid === coll &&? valFilter - ).orderBy(order(pers)) + select(p.all, org.all, c.all), + from(p) + .leftJoin(org, org.oid === p.oid) + .leftJoin(c, c.personId === p.pid), + p.cid === coll &&? valFilter + ).orderBy(order(p)) sql.build .query[(RPerson, Option[ROrganization], Option[RContact])] @@ -101,16 +95,13 @@ object QOrganization { coll: Ident, persId: Ident ): ConnectionIO[Option[(RPerson, Option[ROrganization], Vector[RContact])]] = { - val pers = RPerson.as("p") - val org = ROrganization.as("o") - val c = RContact.as("c") val sql = run( - select(pers.all, org.all, c.all), - from(pers) - .leftJoin(org, pers.oid === org.oid) - .leftJoin(c, c.personId === pers.pid), - pers.cid === coll && pers.pid === persId + select(p.all, org.all, c.all), + from(p) + .leftJoin(org, p.oid === org.oid) + .leftJoin(c, c.personId === p.pid), + p.cid === coll && p.pid === persId ) sql @@ -131,9 +122,7 @@ object QOrganization { value: String, ck: Option[ContactKind], concerning: Option[Boolean] - ): Stream[ConnectionIO, RPerson] = { - val p = RPerson.as("p") - val c = RContact.as("c") + ): Stream[ConnectionIO, RPerson] = runDistinct( select(p.all), from(p).innerJoin(c, c.personId === p.pid), @@ -141,7 +130,6 @@ object QOrganization { concerning.map(c => p.concerning === c) &&? ck.map(k => c.kind === k) ).query[RPerson].stream - } def addOrg[F[_]]( org: ROrganization, From 35c62049f566dc2832d4697a79f031399c6e3646 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 22:56:19 +0100 Subject: [PATCH 16/38] Start converting QItem --- .../docspell/joex/process/CreateItem.scala | 2 +- .../docspell/joex/process/ItemHandler.scala | 2 +- .../scala/docspell/store/qb/CteBind.scala | 7 +- .../main/scala/docspell/store/qb/DSL.scala | 7 + .../main/scala/docspell/store/qb/Select.scala | 44 +++- .../scala/docspell/store/qb/TableDef.scala | 2 +- .../store/qb/impl/CommonBuilder.scala | 1 + .../store/qb/impl/DBFunctionBuilder.scala | 4 +- .../store/qb/impl/SelectBuilder.scala | 18 +- .../scala/docspell/store/queries/QItem.scala | 243 +++++++----------- 10 files changed, 170 insertions(+), 160 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 92d275fa..fe21203b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -121,7 +121,7 @@ object CreateItem { private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = Task { ctx => - val states = ItemState.invalidStates.toList.toSet + val states = ItemState.invalidStates val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet for { cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states)) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 757493d6..c211ce5b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -105,7 +105,7 @@ object ItemHandler { private def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, Args, Unit] = Task { ctx => - val states = ItemState.invalidStates.toList.toSet + val states = ItemState.invalidStates for { items <- ctx.store.transact( QItem.findByFileIds(ctx.args.files.map(_.fileMetaId), states) diff --git a/modules/store/src/main/scala/docspell/store/qb/CteBind.scala b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala index 0a22a056..16e5d436 100644 --- a/modules/store/src/main/scala/docspell/store/qb/CteBind.scala +++ b/modules/store/src/main/scala/docspell/store/qb/CteBind.scala @@ -1,9 +1,12 @@ package docspell.store.qb -case class CteBind(name: TableDef, select: Select) {} +case class CteBind(name: TableDef, coldef: Vector[Column[_]], select: Select) {} object CteBind { def apply(t: (TableDef, Select)): CteBind = - CteBind(t._1, t._2) + CteBind(t._1, Vector.empty, t._2) + + def apply(name: TableDef, col: Column[_], cols: Column[_]*)(select: Select): CteBind = + CteBind(name, cols.toVector.prepended(col), select) } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index dbddbdc9..4a1aa116 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -25,6 +25,13 @@ trait DSL extends DoobieMeta { def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl = DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector) + def withCte( + name: TableDef, + col: Column[_], + cols: Column[_]* + ): Select => DSL.WithCteDsl = + sel => DSL.WithCteDsl(CteBind(name, col, cols: _*)(sel), Vector.empty) + def select(cond: Condition): Nel[SelectExpr] = Nel.of(SelectExpr.SelectCondition(cond, None)) diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index e219ee03..74bd8b6c 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -21,9 +21,21 @@ sealed trait Select { def limit(n: Int): Select = this match { - case Select.Limit(q, _) => Select.Limit(q, n) - case _ => Select.Limit(this, n) + case Select.Limit(q, _) => + Select.Limit(q, n) + case _ => + Select.Limit(this, n) } + + def appendCte(next: CteBind): Select = + this match { + case Select.WithCte(cte, ctes, query) => + Select.WithCte(cte, ctes :+ next, query) + case _ => + Select.WithCte(next, Vector.empty, this) + } + + def appendSelect(e: SelectExpr): Select } object Select { @@ -69,16 +81,34 @@ object Select { copy(where = c) def where(c: Condition): SimpleSelect = copy(where = Some(c)) + + def appendSelect(e: SelectExpr): SimpleSelect = + copy(projection = projection.append(e)) } - case class Union(q: Select, qs: Vector[Select]) extends Select + case class Union(q: Select, qs: Vector[Select]) extends Select { + def appendSelect(e: SelectExpr): Union = + copy(q = q.appendSelect(e)) + } - case class Intersect(q: Select, qs: Vector[Select]) extends Select + case class Intersect(q: Select, qs: Vector[Select]) extends Select { + def appendSelect(e: SelectExpr): Intersect = + copy(q = q.appendSelect(e)) + } case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) - extends Select + extends Select { + def appendSelect(e: SelectExpr): Ordered = + copy(q = q.appendSelect(e)) + } - case class Limit(q: Select, limit: Int) extends Select + case class Limit(q: Select, limit: Int) extends Select { + def appendSelect(e: SelectExpr): Limit = + copy(q = q.appendSelect(e)) + } - case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select + case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select { + def appendSelect(e: SelectExpr): WithCte = + copy(query = query.appendSelect(e)) + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala index 4ef6cfa4..e78d3ff4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/TableDef.scala +++ b/modules/store/src/main/scala/docspell/store/qb/TableDef.scala @@ -8,7 +8,7 @@ trait TableDef { object TableDef { - def apply(table: String, aliasName: Option[String] = None): TableDef = + def apply(table: String, aliasName: Option[String] = None): BasicTable = BasicTable(table, aliasName) final case class BasicTable(tableName: String, alias: Option[String]) extends TableDef { diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala index 8b79cfdf..e0418e60 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala @@ -18,3 +18,4 @@ trait CommonBuilder { def appendAs(alias: Option[String]): Fragment = alias.map(a => fr" AS" ++ Fragment.const(a)).getOrElse(Fragment.empty) } +object CommonBuilder extends CommonBuilder diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index 494ec66c..a805f2dc 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -24,10 +24,10 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Coalesce(expr, exprs) => val v = exprs.prepended(expr).map(SelectExprBuilder.build) - sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ sql")" + sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ fr")" case DBFunction.Power(expr, base) => - sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")" + sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")" case DBFunction.Calc(op, left, right) => SelectExprBuilder.build(left) ++ diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index 23ca286e..40234b96 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -1,9 +1,9 @@ package docspell.store.qb.impl import docspell.store.qb._ - import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} +import cats.data.NonEmptyList object SelectBuilder { val comma = fr"," @@ -74,5 +74,19 @@ object SelectBuilder { } def buildCte(bind: CteBind): Fragment = - Fragment.const(bind.name.tableName) ++ sql"AS (" ++ build(bind.select) ++ sql")" + bind match { + case CteBind(name, cols, select) => + val colDef = + NonEmptyList + .fromFoldable(cols) + .map(nel => + nel + .map(col => CommonBuilder.columnNoPrefix(col)) + .reduceLeft(_ ++ comma ++ _) + ) + .map(f => sql"(" ++ f ++ sql")") + .getOrElse(Fragment.empty) + + Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")" + } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 5c9ca443..db4aa531 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -6,7 +6,6 @@ import cats.effect.Sync import cats.effect.concurrent.Ref import cats.implicits._ import fs2.Stream - import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store @@ -14,7 +13,6 @@ import docspell.store.impl.Implicits._ import docspell.store.impl._ import docspell.store.qb.Select import docspell.store.records._ - import bitpeace.FileMeta import doobie._ import doobie.implicits._ @@ -583,19 +581,18 @@ object QItem { } private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val aPos = RAttachment.Columns.position.prefix("a") - val aName = RAttachment.Columns.name.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mPages = RAttachmentMeta.Columns.pages.prefix("m") + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val cols = Seq(aId, aPos, aName, mPages) - val join = RAttachment.table ++ - fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) - val cond = aItem.is(item) + val a = RAttachment.as("a") + val m = RAttachmentMeta.as("m") - selectSimple(cols, join, cond).query[AttachmentLight].to[List] + Select( + select(a.id, a.position, a.name, m.pages), + from(a) + .leftJoin(m, m.id === a.id), + a.itemId === item + ).build.query[AttachmentLight].to[List] } def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = @@ -609,108 +606,73 @@ object QItem { private def findByFileIdsQuery( fileMetaIds: NonEmptyList[Ident], - limit: Option[Int], - states: Set[ItemState] - ): Fragment = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iState = RItem.Columns.state.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1 = RFileMeta.as("m1") - val m2 = RFileMeta.as("m2") - val m3 = RFileMeta.as("m3") - val m1Id = m1.id.column - val m2Id = m2.id.column - val m3Id = m3.id.column - val filemetaTable = Fragment.const(RFileMeta.T.tableName) + states: Option[NonEmptyList[ItemState]] + ): Select.SimpleSelect = { + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val from = - RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ - fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ - fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) + val i = RItem.as("i") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val r = RAttachmentArchive.as("r") - val fileCond = - or(m1Id.isIn(fileMetaIds), m2Id.isIn(fileMetaIds), m3Id.isIn(fileMetaIds)) - val cond = NonEmptyList.fromList(states.toList) match { - case Some(nel) => - and(fileCond, iState.isIn(nel)) - case None => - fileCond - } - val q = selectSimple(IC, from, cond) - - limit match { - case Some(n) => q ++ fr"LIMIT $n" - case None => q - } + Select( + select(i.all), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(s, s.id === a.id) + .leftJoin(r, r.id === a.id), + (a.fileId.in(fileMetaIds) || + s.fileId.in(fileMetaIds) || + r.fileId.in(fileMetaIds)) &&? states.map(nel => i.state.in(nel)) + ) } def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] = NonEmptyList.fromList(fileMetaIds.toList) match { case Some(nel) => - findByFileIdsQuery(nel, Some(1), Set.empty).query[RItem].option + findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option case None => (None: Option[RItem]).pure[ConnectionIO] } def findByFileIds( fileMetaIds: Seq[Ident], - states: Set[ItemState] + states: NonEmptyList[ItemState] ): ConnectionIO[Vector[RItem]] = NonEmptyList.fromList(fileMetaIds.toList) match { case Some(nel) => - findByFileIdsQuery(nel, None, states).query[RItem].to[Vector] + findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector] case None => Vector.empty[RItem].pure[ConnectionIO] } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val aItem = RAttachment.Columns.itemId.prefix("a") - val aId = RAttachment.Columns.id.prefix("a") - val aFileId = RAttachment.Columns.fileId.prefix("a") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") - val sId = RAttachmentSource.Columns.id.prefix("s") - val sFileId = RAttachmentSource.Columns.fileId.prefix("s") - val rId = RAttachmentArchive.Columns.id.prefix("r") - val rFileId = RAttachmentArchive.Columns.fileId.prefix("r") - val m1 = RFileMeta.as("m1") - val m2 = RFileMeta.as("m2") - val m3 = RFileMeta.as("m3") - val m1Id = m1.id.column - val m2Id = m2.id.column - val m3Id = m3.id.column - val m1Checksum = m1.checksum.column - val m2Checksum = m2.checksum.column - val m3Checksum = m3.checksum.column - val filemetaTable = Fragment.const(RFileMeta.T.tableName) - val from = - RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ - fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++ - fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++ - fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++ - fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId) + import docspell.store.qb._ + import docspell.store.qb.DSL._ - selectSimple( - IC, - from, - and( - or(m1Checksum.is(checksum), m2Checksum.is(checksum), m3Checksum.is(checksum)), - iColl.is(collective) + val m1 = RFileMeta.as("m1") + val m2 = RFileMeta.as("m2") + val m3 = RFileMeta.as("m3") + val i = RItem.as("i") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val r = RAttachmentArchive.as("r") + + Select( + select(i.all), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(s, s.id === a.id) + .innerJoin(m1, m1.id === a.fileId) + .innerJoin(m2, m2.id === s.fileId) + .leftJoin(r, r.id === a.id) + .leftJoin(m3, m3.id === r.fileId), + where( + i.cid === collective && + (m1.checksum === checksum || m2.checksum === checksum || m3.checksum === checksum) ) - ).query[RItem] - .to[Vector] + ).build.query[RItem].to[Vector] } final case class NameAndNotes( @@ -724,15 +686,16 @@ object QItem { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, NameAndNotes] = { - val iId = RItem.Columns.id - val iColl = RItem.Columns.cid - val iName = RItem.Columns.name - val iFolder = RItem.Columns.folder - val iNotes = RItem.Columns.notes + import docspell.store.qb._ + import docspell.store.qb.DSL._ - val cols = Seq(iId, iColl, iFolder, iName, iNotes) - val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty) - selectSimple(cols, RItem.table, where) + val i = RItem.as("i") + + Select( + select(i.id, i.cid, i.folder, i.name, i.notes), + from(i) + ).where(coll.map(cid => i.cid === cid)) + .build .query[NameAndNotes] .streamWithChunkSize(chunkSize) } @@ -741,15 +704,13 @@ object QItem { collective: Ident, chunkSize: Int ): Stream[ConnectionIO, Ident] = { - val cols = Seq(RItem.Columns.id) - val iColl = RItem.Columns.cid - val iState = RItem.Columns.state - (selectSimple( - cols, - RItem.table, - and(iColl.is(collective), iState.is(ItemState.confirmed)) - ) ++ - orderBy(RItem.Columns.created.desc)) + import docspell.store.qb._ + import docspell.store.qb.DSL._ + + val i = RItem.as("i") + Select(i.id.s, from(i), i.cid === collective && i.state === ItemState.confirmed) + .orderBy(i.created.desc) + .build .query[Ident] .streamWithChunkSize(chunkSize) } @@ -763,45 +724,39 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - val aId = RAttachment.Columns.id.prefix("a") - val aItem = RAttachment.Columns.itemId.prefix("a") - val mId = RAttachmentMeta.Columns.id.prefix("m") - val mText = RAttachmentMeta.Columns.content.prefix("m") - val tagItem = RTagItem.as("ti") //Columns.itemId.prefix("ti") - //val tiTag = RTagItem.Columns.tagId.prefix("ti") + import docspell.store.qb._ + import docspell.store.qb.DSL._ + val tag = RTag.as("t") -// val tId = RTag.Columns.tid.prefix("t") -// val tName = RTag.Columns.name.prefix("t") -// val tCat = RTag.Columns.category.prefix("t") - val iId = RItem.Columns.id.prefix("i") - val iColl = RItem.Columns.cid.prefix("i") + val a = RAttachment.as("a") + val am = RAttachmentMeta.as("m") + val ti = RTagItem.as("ti") + val i = RItem.as("i") - val cte = withCTE( - "tags" -> selectSimple( - Seq(tagItem.itemId.column, tag.tid.column, tag.name.column), - Fragment.const(RTagItem.t.tableName) ++ fr"ti INNER JOIN" ++ - Fragment.const(tag.tableName) ++ fr"t ON" ++ tag.tid.column - .is(tagItem.tagId.column), - and(tagItem.itemId.column.is(itemId), tag.category.column.is(tagCategory)) - ) - ) + val tags = TableDef("tags").as("tt") + val tagsItem = Column[Ident]("itemid", tags) + val tagsTid = Column[Ident]("tid", tags) + val tagsName = Column[String]("tname", tags) - val cols = Seq(mText, tag.tid.column, tag.name.column) + val q = + withCte( + tags -> Select( + select(ti.itemId.as(tagsItem), tag.tid.as(tagsTid), tag.name.as(tagsName)), + from(ti) + .innerJoin(tag, tag.tid === ti.tagId), + ti.itemId === itemId && tag.category === tagCategory + ) + )( + Select( + select(am.content, tagsTid, tagsName), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(am, a.id === am.id) + .leftJoin(tags, tagsItem === i.id), + i.id === itemId && i.cid === collective && am.content.isNotNull && am.content <> "" + ) + ).build - val from = RItem.table ++ fr"i INNER JOIN" ++ - RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++ - RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"LEFT JOIN" ++ - fr"tags t ON" ++ RTagItem.t.itemId.oldColumn.prefix("t").is(iId) - - val where = - and( - iId.is(itemId), - iColl.is(collective), - mText.isNotNull, - mText.isNot("") - ) - - val q = cte ++ selectDistinct(cols, from, where) for { _ <- logger.ftrace[ConnectionIO]( s"query: $q (${itemId.id}, ${collective.id}, ${tagCategory})" From 5e2c5d2a50c2d10a4811a0ea06e6ddc941583df4 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 13 Dec 2020 23:44:46 +0100 Subject: [PATCH 17/38] Extends query builder --- .../docspell/joex/analysis/RegexNerFile.scala | 2 +- .../main/scala/docspell/store/qb/DSL.scala | 4 +- .../scala/docspell/store/qb/FromExpr.scala | 47 +++++++++++++++---- .../main/scala/docspell/store/qb/Join.scala | 10 ---- .../main/scala/docspell/store/qb/Select.scala | 5 ++ .../store/qb/impl/FromExprBuilder.scala | 30 +++++++----- .../store/qb/impl/SelectBuilder.scala | 3 ++ .../scala/docspell/store/queries/QJob.scala | 2 +- 8 files changed, 69 insertions(+), 34 deletions(-) delete mode 100644 modules/store/src/main/scala/docspell/store/qb/Join.scala diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 75c21673..24e7f6ae 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -150,7 +150,7 @@ object RegexNerFile { ) val t = Column[Timestamp]("t", TableDef("")) - run(select(max(t)), fromSubSelect(sql).as("x")) + run(select(max(t)), from(sql, "x")) .query[Option[Timestamp]] .option .map(_.flatten) diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 4a1aa116..45905d84 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -59,8 +59,8 @@ trait DSL extends DoobieMeta { def from(table: TableDef): FromExpr.From = FromExpr.From(table) - def fromSubSelect(sel: Select): FromExpr.SubSelect = - FromExpr.SubSelect(sel, "x") + def from(sel: Select, alias: String): FromExpr.From = + FromExpr.From(sel, alias) def count(c: Column[_]): DBFunction = DBFunction.Count(c) diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index 8a8d65b1..ac32d791 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -4,24 +4,55 @@ sealed trait FromExpr object FromExpr { - case class From(table: TableDef) extends FromExpr { - def innerJoin(other: TableDef, on: Condition): Joined = + case class From(table: Relation) extends FromExpr { + def innerJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.InnerJoin(other, on))) - def leftJoin(other: TableDef, on: Condition): Joined = + def innerJoin(other: TableDef, on: Condition): Joined = + innerJoin(Relation.Table(other), on) + + def leftJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.LeftJoin(other, on))) + + def leftJoin(other: TableDef, on: Condition): Joined = + leftJoin(Relation.Table(other), on) + } + + object From { + def apply(td: TableDef): From = + From(Relation.Table(td)) + + def apply(select: Select, alias: String): From = + From(Relation.SubSelect(select, alias)) } case class Joined(from: From, joins: Vector[Join]) extends FromExpr { - def innerJoin(other: TableDef, on: Condition): Joined = + def innerJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.InnerJoin(other, on)) - def leftJoin(other: TableDef, on: Condition): Joined = + def innerJoin(other: TableDef, on: Condition): Joined = + innerJoin(Relation.Table(other), on) + + def leftJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.LeftJoin(other, on)) + + def leftJoin(other: TableDef, on: Condition): Joined = + leftJoin(Relation.Table(other), on) } - case class SubSelect(sel: Select, name: String) extends FromExpr { - def as(name: String): SubSelect = - copy(name = name) + sealed trait Relation + object Relation { + final case class Table(table: TableDef) extends Relation + final case class SubSelect(select: Select, alias: String) extends Relation { + def as(a: String): SubSelect = + copy(alias = a) + } } + + sealed trait Join + object Join { + final case class InnerJoin(table: Relation, cond: Condition) extends Join + final case class LeftJoin(table: Relation, cond: Condition) extends Join + } + } diff --git a/modules/store/src/main/scala/docspell/store/qb/Join.scala b/modules/store/src/main/scala/docspell/store/qb/Join.scala deleted file mode 100644 index a51a3b70..00000000 --- a/modules/store/src/main/scala/docspell/store/qb/Join.scala +++ /dev/null @@ -1,10 +0,0 @@ -package docspell.store.qb - -sealed trait Join - -object Join { - - case class InnerJoin(table: TableDef, cond: Condition) extends Join - - case class LeftJoin(table: TableDef, cond: Condition) extends Join -} diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index 74bd8b6c..f1135f97 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -86,6 +86,11 @@ object Select { copy(projection = projection.append(e)) } + case class RawSelect(fragment: Fragment) extends Select { + def appendSelect(e: SelectExpr): RawSelect = + sys.error("RawSelect doesn't support appending select expressions") + } + case class Union(q: Select, qs: Vector[Select]) extends Select { def appendSelect(e: SelectExpr): Union = copy(q = q.appendSelect(e)) diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala index 926ed330..522a8941 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/FromExprBuilder.scala @@ -9,15 +9,12 @@ object FromExprBuilder { def build(expr: FromExpr): Fragment = expr match { - case FromExpr.From(table) => - fr" FROM" ++ buildTable(table) + case FromExpr.From(relation) => + fr" FROM" ++ buildRelation(relation) case FromExpr.Joined(from, joins) => build(from) ++ joins.map(buildJoin).foldLeft(Fragment.empty)(_ ++ _) - - case FromExpr.SubSelect(sel, name) => - sql" FROM (" ++ SelectBuilder(sel) ++ fr") AS" ++ Fragment.const(name) } def buildTable(table: TableDef): Fragment = @@ -25,15 +22,24 @@ object FromExprBuilder { .map(a => Fragment.const0(a)) .getOrElse(Fragment.empty) - def buildJoin(join: Join): Fragment = - join match { - case Join.InnerJoin(table, cond) => - val c = fr" ON" ++ ConditionBuilder.build(cond) - fr" INNER JOIN" ++ buildTable(table) ++ c + def buildRelation(rel: FromExpr.Relation): Fragment = + rel match { + case FromExpr.Relation.Table(table) => + buildTable(table) - case Join.LeftJoin(table, cond) => + case FromExpr.Relation.SubSelect(sel, alias) => + sql" (" ++ SelectBuilder(sel) ++ fr") AS" ++ Fragment.const(alias) + } + + def buildJoin(join: FromExpr.Join): Fragment = + join match { + case FromExpr.Join.InnerJoin(table, cond) => val c = fr" ON" ++ ConditionBuilder.build(cond) - fr" LEFT JOIN" ++ buildTable(table) ++ c + fr" INNER JOIN" ++ buildRelation(table) ++ c + + case FromExpr.Join.LeftJoin(table, cond) => + val c = fr" ON" ++ ConditionBuilder.build(cond) + fr" LEFT JOIN" ++ buildRelation(table) ++ c } } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index 40234b96..ee50ba67 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -21,6 +21,9 @@ object SelectBuilder { val sel = if (sq.distinctFlag) fr"SELECT DISTINCT" else fr"SELECT" sel ++ buildSimple(sq) + case Select.RawSelect(f) => + f + case Select.Union(q, qs) => qs.prepended(q).map(build).reduce(_ ++ union ++ _) diff --git a/modules/store/src/main/scala/docspell/store/queries/QJob.scala b/modules/store/src/main/scala/docspell/store/queries/QJob.scala index c85080a1..1d7b4c3e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QJob.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QJob.scala @@ -110,7 +110,7 @@ object QJob { val gcol = Column[String]("g", TableDef("")) val groups = - Select(select(gcol), fromSubSelect(union(sql1, sql2)).as("t0"), gcol.isNull.negate) + Select(select(gcol), from(union(sql1, sql2), "t0"), gcol.isNull.negate) // either 0, one or two results, but may be empty if RJob table is empty groups.build.query[Ident].to[List].map(_.headOption) From 266fec9eb58db3b4fac8515d3dc61d6608f7335d Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 11:50:35 +0100 Subject: [PATCH 18/38] Convert find items query --- .../docspell/backend/ops/OFulltext.scala | 22 +- .../docspell/backend/ops/OItemSearch.scala | 6 +- .../store/{queries => qb}/Batch.scala | 2 +- .../scala/docspell/store/qb/Condition.scala | 34 +- .../scala/docspell/store/qb/DBFunction.scala | 2 + .../main/scala/docspell/store/qb/DSL.scala | 19 +- .../scala/docspell/store/qb/FromExpr.scala | 28 +- .../scala/docspell/store/qb/OrderBy.scala | 6 + .../main/scala/docspell/store/qb/Select.scala | 80 +++- .../store/qb/impl/DBFunctionBuilder.scala | 3 + .../store/qb/impl/SelectBuilder.scala | 24 +- .../scala/docspell/store/queries/QItem.scala | 395 ++++++++---------- .../store/queries/QueryWildcard.scala | 3 + .../docspell/store/records/RAttachment.scala | 151 +------ .../docspell/store/records/RTagItem.scala | 20 +- .../docspell/store/records/TagItemName.scala | 87 +++- .../docspell/store/qb/QueryBuilderTest.scala | 4 +- .../docspell/store/qb/impl/DSLTest.scala | 19 + 18 files changed, 455 insertions(+), 450 deletions(-) rename modules/store/src/main/scala/docspell/store/{queries => qb}/Batch.scala (92%) create mode 100644 modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 9bc3f4a6..f9efed22 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -8,30 +8,30 @@ import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.ftsclient._ -import docspell.store.Store import docspell.store.queries.{QFolder, QItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob +import docspell.store.{Store, qb} trait OFulltext[F[_]] { def findItems(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItem]] /** Same as `findItems` but does more queries per item to find all tags. */ def findItemsWithTags(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] def findIndexOnly(maxNoteLen: Int)( fts: OFulltext.FtsInput, account: AccountId, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] /** Clears the full-text index completely and launches a task that @@ -95,7 +95,7 @@ object OFulltext { def findIndexOnly(maxNoteLen: Int)( ftsQ: OFulltext.FtsInput, account: AccountId, - batch: Batch + batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] = { val fq = FtsQuery( ftsQ.query, @@ -135,7 +135,7 @@ object OFulltext { def findItems( maxNoteLen: Int - )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] = + )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] = findItemsFts( q, ftsQ, @@ -152,7 +152,7 @@ object OFulltext { def findItemsWithTags(maxNoteLen: Int)( q: Query, ftsQ: FtsInput, - batch: Batch + batch: qb.Batch ): F[Vector[FtsItemWithTags]] = findItemsFts( q, @@ -172,8 +172,8 @@ object OFulltext { private def findItemsFts[A: ItemId, B]( q: Query, ftsQ: FtsInput, - batch: Batch, - search: (Query, Batch) => F[Vector[A]], + batch: qb.Batch, + search: (Query, qb.Batch) => F[Vector[A]], convert: ( FtsResult, Map[Ident, List[FtsResult.ItemMatch]] @@ -186,8 +186,8 @@ object OFulltext { private def findItemsFts0[A: ItemId, B]( q: Query, ftsQ: FtsInput, - batch: Batch, - search: (Query, Batch) => F[Vector[A]], + batch: qb.Batch, + search: (Query, qb.Batch) => F[Vector[A]], convert: ( FtsResult, Map[Ident, List[FtsResult.ItemMatch]] diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 25efda8d..94270e6f 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -7,9 +7,9 @@ import fs2.Stream import docspell.backend.ops.OItemSearch._ import docspell.common._ -import docspell.store.Store import docspell.store.queries.{QAttachment, QItem} import docspell.store.records._ +import docspell.store.{Store, qb} import bitpeace.{FileMeta, RangeDef} import doobie.implicits._ @@ -59,8 +59,8 @@ object OItemSearch { type Query = QItem.Query val Query = QItem.Query - type Batch = docspell.store.queries.Batch - val Batch = docspell.store.queries.Batch + type Batch = qb.Batch + val Batch = docspell.store.qb.Batch type ListItem = QItem.ListItem val ListItem = QItem.ListItem diff --git a/modules/store/src/main/scala/docspell/store/queries/Batch.scala b/modules/store/src/main/scala/docspell/store/qb/Batch.scala similarity index 92% rename from modules/store/src/main/scala/docspell/store/queries/Batch.scala rename to modules/store/src/main/scala/docspell/store/qb/Batch.scala index d88ec957..26ffa148 100644 --- a/modules/store/src/main/scala/docspell/store/queries/Batch.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Batch.scala @@ -1,4 +1,4 @@ -package docspell.store.queries +package docspell.store.qb case class Batch(offset: Int, limit: Int) { def restrictLimitTo(n: Int): Batch = diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index c89b7178..2a1f3097 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -26,8 +26,36 @@ object Condition { case class IsNull(col: Column[_]) extends Condition - case class And(c: Condition, cs: Vector[Condition]) extends Condition - case class Or(c: Condition, cs: Vector[Condition]) extends Condition - case class Not(c: Condition) extends Condition + case class And(c: Condition, cs: Vector[Condition]) extends Condition { + def append(other: Condition): And = + other match { + case And(oc, ocs) => + And(c, cs ++ (oc +: ocs)) + case _ => + And(c, cs :+ other) + } + } + object And { + def apply(c: Condition, cs: Condition*): And = + And(c, cs.toVector) + + } + + case class Or(c: Condition, cs: Vector[Condition]) extends Condition { + def append(other: Condition): Or = + other match { + case Or(oc, ocs) => + Or(c, cs ++ (oc +: ocs)) + case _ => + Or(c, cs :+ other) + } + } + object Or { + def apply(c: Condition, cs: Condition*): Or = + Or(c, cs.toVector) + } + + case class Not(c: Condition) extends Condition + object Not {} } diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 24de3520..86ad7b1a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -23,6 +23,8 @@ object DBFunction { case class Calc(op: Operator, left: SelectExpr, right: SelectExpr) extends DBFunction + case class Substring(expr: SelectExpr, start: Int, length: Int) extends DBFunction + sealed trait Operator object Operator { case object Plus extends Operator diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 45905d84..97c80a2a 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -80,6 +80,9 @@ trait DSL extends DoobieMeta { def power(base: Int, expr: SelectExpr): DBFunction = DBFunction.Power(expr, base) + def substring(expr: SelectExpr, start: Int, length: Int): DBFunction = + DBFunction.Substring(expr, start, length) + def lit[A](value: A)(implicit P: Put[A]): SelectExpr.SelectLit[A] = SelectExpr.SelectLit(value, None) @@ -91,8 +94,8 @@ trait DSL extends DoobieMeta { def and(c: Condition, cs: Condition*): Condition = c match { - case Condition.And(head, tail) => - Condition.And(head, tail ++ (c +: cs.toVector)) + case a: Condition.And => + cs.foldLeft(a)(_.append(_)) case _ => Condition.And(c, cs.toVector) } @@ -173,12 +176,21 @@ trait DSL extends DoobieMeta { def in(subsel: Select): Condition = Condition.InSubSelect(col, subsel) + def notIn(subsel: Select): Condition = + in(subsel).negate + def in(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, false) + def notIn(values: Nel[A])(implicit P: Put[A]): Condition = + in(values).negate + def inLower(values: Nel[A])(implicit P: Put[A]): Condition = Condition.InValues(col, values, true) + def notInLower(values: Nel[A])(implicit P: Put[A]): Condition = + Condition.InValues(col, values, true).negate + def isNull: Condition = Condition.IsNull(col) @@ -224,6 +236,9 @@ trait DSL extends DoobieMeta { def as(alias: String): SelectExpr = SelectExpr.SelectFun(dbf, Some(alias)) + def as(otherCol: Column[_]): SelectExpr = + SelectExpr.SelectFun(dbf, Some(otherCol.name)) + def ===[A](value: A)(implicit P: Put[A]): Condition = Condition.CompareFVal(dbf, Operator.Eq, value) diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index ac32d791..72486323 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -1,6 +1,21 @@ package docspell.store.qb -sealed trait FromExpr +import docspell.store.qb.FromExpr.{Joined, Relation} + +sealed trait FromExpr { + def innerJoin(other: Relation, on: Condition): Joined + + def innerJoin(other: TableDef, on: Condition): Joined = + innerJoin(Relation.Table(other), on) + + def leftJoin(other: Relation, on: Condition): Joined + + def leftJoin(other: TableDef, on: Condition): Joined = + leftJoin(Relation.Table(other), on) + + def leftJoin(sel: Select, alias: String, on: Condition): Joined = + leftJoin(Relation.SubSelect(sel, alias), on) +} object FromExpr { @@ -8,14 +23,8 @@ object FromExpr { def innerJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.InnerJoin(other, on))) - def innerJoin(other: TableDef, on: Condition): Joined = - innerJoin(Relation.Table(other), on) - def leftJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.LeftJoin(other, on))) - - def leftJoin(other: TableDef, on: Condition): Joined = - leftJoin(Relation.Table(other), on) } object From { @@ -30,14 +39,9 @@ object FromExpr { def innerJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.InnerJoin(other, on)) - def innerJoin(other: TableDef, on: Condition): Joined = - innerJoin(Relation.Table(other), on) - def leftJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.LeftJoin(other, on)) - def leftJoin(other: TableDef, on: Condition): Joined = - leftJoin(Relation.Table(other), on) } sealed trait Relation diff --git a/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala index 24d42cd4..128d3b81 100644 --- a/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala +++ b/modules/store/src/main/scala/docspell/store/qb/OrderBy.scala @@ -6,6 +6,12 @@ final case class OrderBy(expr: SelectExpr, orderType: OrderType) object OrderBy { + def asc(e: SelectExpr): OrderBy = + OrderBy(e, OrderType.Asc) + + def desc(e: SelectExpr): OrderBy = + OrderBy(e, OrderType.Desc) + sealed trait OrderType object OrderType { case object Asc extends OrderType diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index f1135f97..155109a8 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -13,20 +13,22 @@ sealed trait Select { def as(alias: String): SelectExpr.SelectQuery = SelectExpr.SelectQuery(this, Some(alias)) - def orderBy(ob: OrderBy, obs: OrderBy*): Select.Ordered = - Select.Ordered(this, ob, obs.toVector) + def orderBy(ob: OrderBy, obs: OrderBy*): Select - def orderBy(c: Column[_]): Select.Ordered = + def orderBy(c: Column[_]): Select = orderBy(OrderBy(SelectExpr.SelectColumn(c, None), OrderBy.OrderType.Asc)) - def limit(n: Int): Select = + def limit(batch: Batch): Select = this match { case Select.Limit(q, _) => - Select.Limit(q, n) + Select.Limit(q, batch) case _ => - Select.Limit(this, n) + Select.Limit(this, batch) } + def limit(n: Int): Select = + limit(Batch.limit(n)) + def appendCte(next: CteBind): Select = this match { case Select.WithCte(cte, ctes, query) => @@ -36,6 +38,10 @@ sealed trait Select { } def appendSelect(e: SelectExpr): Select + + def changeFrom(f: FromExpr => FromExpr): Select + + def changeWhere(f: Condition => Condition): Select } object Select { @@ -84,36 +90,96 @@ object Select { def appendSelect(e: SelectExpr): SimpleSelect = copy(projection = projection.append(e)) + + def changeFrom(f: FromExpr => FromExpr): SimpleSelect = + copy(from = f(from)) + + def changeWhere(f: Condition => Condition): SimpleSelect = + copy(where = where.map(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Select = + Ordered(this, ob, obs.toVector) } case class RawSelect(fragment: Fragment) extends Select { def appendSelect(e: SelectExpr): RawSelect = sys.error("RawSelect doesn't support appending select expressions") + + def changeFrom(f: FromExpr => FromExpr): Select = + sys.error("RawSelect doesn't support changing from expression") + + def changeWhere(f: Condition => Condition): Select = + sys.error("RawSelect doesn't support changing where condition") + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + sys.error("RawSelect doesn't support adding orderBy clause") } case class Union(q: Select, qs: Vector[Select]) extends Select { def appendSelect(e: SelectExpr): Union = copy(q = q.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): Union = + copy(q = q.changeFrom(f)) + + def changeWhere(f: Condition => Condition): Union = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(this, ob, obs.toVector) } case class Intersect(q: Select, qs: Vector[Select]) extends Select { def appendSelect(e: SelectExpr): Intersect = copy(q = q.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): Intersect = + copy(q = q.changeFrom(f)) + + def changeWhere(f: Condition => Condition): Intersect = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(this, ob, obs.toVector) } case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) extends Select { def appendSelect(e: SelectExpr): Ordered = copy(q = q.appendSelect(e)) + def changeFrom(f: FromExpr => FromExpr): Ordered = + copy(q = q.changeFrom(f)) + def changeWhere(f: Condition => Condition): Ordered = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = + Ordered(q, ob, obs.toVector) } - case class Limit(q: Select, limit: Int) extends Select { + case class Limit(q: Select, batch: Batch) extends Select { def appendSelect(e: SelectExpr): Limit = copy(q = q.appendSelect(e)) + def changeFrom(f: FromExpr => FromExpr): Limit = + copy(q = q.changeFrom(f)) + def changeWhere(f: Condition => Condition): Limit = + copy(q = q.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): Limit = + copy(q = q.orderBy(ob, obs: _*)) + } case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select { def appendSelect(e: SelectExpr): WithCte = copy(query = query.appendSelect(e)) + + def changeFrom(f: FromExpr => FromExpr): WithCte = + copy(query = query.changeFrom(f)) + + def changeWhere(f: Condition => Condition): WithCte = + copy(query = query.changeWhere(f)) + + def orderBy(ob: OrderBy, obs: OrderBy*): WithCte = + copy(query = query.orderBy(ob, obs: _*)) } } diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index a805f2dc..d99302e8 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -29,6 +29,9 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Power(expr, base) => sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")" + case DBFunction.Substring(expr, start, len) => + sql"SUBSTRING(" ++ SelectExprBuilder.build(expr) ++ fr" FROM $start FOR $len)" + case DBFunction.Calc(op, left, right) => SelectExprBuilder.build(left) ++ buildOperator(op) ++ diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index ee50ba67..68743737 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -1,16 +1,18 @@ package docspell.store.qb.impl +import cats.data.NonEmptyList + import docspell.store.qb._ + import _root_.doobie.implicits._ import _root_.doobie.{Query => _, _} -import cats.data.NonEmptyList object SelectBuilder { val comma = fr"," val asc = fr" ASC" val desc = fr" DESC" - val intersect = fr"INTERSECT" - val union = fr"UNION ALL" + val intersect = fr" INTERSECT" + val union = fr" UNION ALL" def apply(q: Select): Fragment = build(q) @@ -34,8 +36,8 @@ object SelectBuilder { val order = obs.prepended(ob).map(orderBy).reduce(_ ++ comma ++ _) build(q) ++ fr" ORDER BY" ++ order - case Select.Limit(q, n) => - build(q) ++ fr" LIMIT $n" + case Select.Limit(q, batch) => + build(q) ++ buildBatch(batch) case Select.WithCte(cte, moreCte, query) => val ctes = moreCte.prepended(cte) @@ -92,4 +94,16 @@ object SelectBuilder { Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")" } + + def buildBatch(b: Batch): Fragment = { + val limitFrag = + if (b.limit != Int.MaxValue) fr" LIMIT ${b.limit}" + else Fragment.empty + + val offsetFrag = + if (b.offset != 0) fr" OFFSET ${b.offset}" + else Fragment.empty + + limitFrag ++ offsetFrag + } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index db4aa531..cea2177e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -1,18 +1,19 @@ package docspell.store.queries -import cats.data.NonEmptyList import cats.data.OptionT +import cats.data.{NonEmptyList => Nel} import cats.effect.Sync import cats.effect.concurrent.Ref import cats.implicits._ import fs2.Stream + import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store -import docspell.store.impl.Implicits._ -import docspell.store.impl._ -import docspell.store.qb.Select +import docspell.store.impl.DoobieMeta._ +import docspell.store.qb._ import docspell.store.records._ + import bitpeace.FileMeta import doobie._ import doobie.implicits._ @@ -86,6 +87,8 @@ object QItem { } def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { + import docspell.store.impl.Implicits._ + val equip = REquipment.as("e") val org = ROrganization.as("o") val pers0 = RPerson.as("p0") @@ -226,7 +229,7 @@ object QItem { itemIds: Option[Set[Ident]], customValues: Seq[CustomValue], source: Option[String], - orderAsc: Option[RItem.Columns.type => Column] + orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]] ) object Query { @@ -260,7 +263,7 @@ object QItem { private def findCustomFieldValuesForColl( coll: Ident, values: Seq[CustomValue] - ): Seq[(String, Fragment)] = { + ): Option[Select] = { import docspell.store.qb.DSL._ val cf = RCustomField.as("cf") @@ -277,122 +280,90 @@ object QItem { ) ) - NonEmptyList.fromList(values.toList) match { - case Some(nel) => - Seq("customvalues" -> intersect(nel.map(singleSelect)).build) - case None => - Seq.empty - } + Nel + .fromList(values.toList) + .map(nel => intersect(nel.map(singleSelect))) } - private def findItemsBase( - q: Query, - distinct: Boolean, - noteMaxLen: Int, - moreCols: Seq[Fragment], - ctes: (String, Fragment)* - ): Fragment = { + private def findItemsBase(q: Query, noteMaxLen: Int): Select = { + import docspell.store.qb.DSL._ + + object Attachs extends TableDef { + val tableName = "attachs" + val aliasName = "cta" + val alias = Some(aliasName) + val num = Column[Int]("num", this) + val itemId = Column[Ident]("item_id", this) + } val equip = REquipment.as("e1") val org = ROrganization.as("o0") - val pers0 = RPerson.as("p0") - val pers1 = RPerson.as("p1") + val p0 = RPerson.as("p0") + val p1 = RPerson.as("p1") val f = RFolder.as("f1") - val cv = RCustomFieldValue.as("cv") + val i = RItem.as("i") + val a = RAttachment.as("a") - val IC = RItem.Columns - val AC = RAttachment.Columns - val itemCols = IC.all - val equipCols = List(equip.eid.oldColumn, equip.name.oldColumn) - val folderCols = List(f.id.oldColumn, f.name.oldColumn) - val cvItem = cv.itemId.column + val coll = q.account.collective - val finalCols = commas( - Seq( - IC.id.prefix("i").f, - IC.name.prefix("i").f, - IC.state.prefix("i").f, - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f), - IC.dueDate.prefix("i").f, - IC.source.prefix("i").f, - IC.incoming.prefix("i").f, - IC.created.prefix("i").f, - fr"COALESCE(a.num, 0)", - org.oid.column.f, - org.name.column.f, - pers0.pid.column.f, - pers0.name.column.f, - pers1.pid.column.f, - pers1.name.column.f, - equip.eid.oldColumn.prefix("e1").f, - equip.name.oldColumn.prefix("e1").f, - f.id.column.f, - f.name.column.f, - // sql uses 1 for first character - IC.notes.prefix("i").substring(1, noteMaxLen), - // last column is only for sorting - q.orderAsc match { - case Some(co) => - coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) - case None => - IC.created.prefix("i").f - } - ) ++ moreCols + val baseSelect = Select( + select( + i.id.s, + i.name.s, + i.state.s, + coalesce(i.itemDate.s, i.created.s).s, + i.dueDate.s, + i.source.s, + i.incoming.s, + i.created.s, + coalesce(Attachs.num.s, lit(0)).s, + org.oid.s, + org.name.s, + p0.pid.s, + p0.name.s, + p1.pid.s, + p1.name.s, + equip.eid.s, + equip.name.s, + f.id.s, + f.name.s, + substring(i.notes.s, 1, noteMaxLen).s, + q.orderAsc + .map(of => coalesce(of(i).s, i.created.s).s) + .getOrElse(i.created.s) + ), + from(i) + .leftJoin(f, f.id === i.folder && f.collective === coll) + .leftJoin( + Select( + select(countAll.as(Attachs.num), a.itemId.as(Attachs.itemId)), + from(a) + .innerJoin(i, i.id === a.itemId), + i.cid === q.account.collective, + GroupBy(a.itemId) + ), + Attachs.aliasName, //alias, todo improve dsl + Attachs.itemId === i.id + ) + .leftJoin(p0, p0.pid === i.corrPerson && p0.cid === coll) + .leftJoin(org, org.oid === i.corrOrg && org.cid === coll) + .leftJoin(p1, p1.pid === i.concPerson && p1.cid === coll) + .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll), + where( + i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) && + or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account))) + ) + ).distinct.orderBy( + q.orderAsc + .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s)) + .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s)) ) - val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.account.collective)) - val withPerson = - selectSimple( - List(RPerson.T.pid.column, RPerson.T.name.column), - Fragment.const(RPerson.T.tableName), - RPerson.T.cid.column.is(q.account.collective) - ) - val withOrgs = - selectSimple( - List(ROrganization.T.oid.column, ROrganization.T.name.column), - Fragment.const(ROrganization.T.tableName), - ROrganization.T.cid.column.is(q.account.collective) - ) - val withEquips = - selectSimple( - equipCols, - Fragment.const(equip.tableName), - equip.cid.oldColumn.is(q.account.collective) - ) - val withFolder = - selectSimple( - folderCols, - Fragment.const(f.tableName), - f.collective.oldColumn.is(q.account.collective) - ) - val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ - fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" - - val withCustomValues = - findCustomFieldValuesForColl(q.account.collective, q.customValues) - - val selectKW = if (distinct) fr"SELECT DISTINCT" else fr"SELECT" - withCTE( - (Seq( - "items" -> withItem, - "persons" -> withPerson, - "orgs" -> withOrgs, - "equips" -> withEquips, - "attachs" -> withAttach, - "folders" -> withFolder - ) ++ withCustomValues ++ ctes): _* - ) ++ - selectKW ++ finalCols ++ fr" FROM items i" ++ - fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ - fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(pers0.pid.column) ++ - fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(org.oid.column) ++ - fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(pers1.pid.column) ++ - fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment - .prefix("i") - .is(equip.eid.oldColumn.prefix("e1")) ++ - fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(f.id.column) ++ - (if (q.customValues.isEmpty) Fragment.empty - else - fr"INNER JOIN customvalues cv ON" ++ cvItem.is(IC.id.prefix("i"))) + findCustomFieldValuesForColl(coll, q.customValues) match { + case Some(itemIds) => + baseSelect.changeWhere(c => c && i.id.in(itemIds)) + case None => + baseSelect + } } def findItems( @@ -400,102 +371,54 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { + import docspell.store.qb.DSL._ + val equip = REquipment.as("e1") val org = ROrganization.as("o0") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") val f = RFolder.as("f1") - val IC = RItem.Columns + val i = RItem.as("i") - // inclusive tags are AND-ed - val tagSelectsIncl = q.tagsInclude - .map(tid => - selectSimple( - List(RTagItem.t.itemId.column), - Fragment.const(RTagItem.t.tableName), - RTagItem.t.tagId.column.is(tid) - ) - ) ++ q.tagCategoryIncl.map(cat => TagItemName.itemsInCategory(NonEmptyList.of(cat))) + val cond: Condition => Condition = + c => + c &&? + q.direction.map(d => i.incoming === d) &&? + q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? + q.allNames + .map(QueryWildcard.lower) + .map(n => + org.name.like(n) || + pers0.name.like(n) || + pers1.name.like(n) || + equip.name.like(n) || + i.name.like(n) || + i.notes.like(n) + ) &&? + q.corrPerson.map(p => pers0.pid === p) &&? + q.corrOrg.map(o => org.oid === o) &&? + q.concPerson.map(p => pers1.pid === p) &&? + q.concEquip.map(e => equip.eid === e) &&? + q.folder.map(fid => f.id === fid) &&? + q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&? + q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&? + q.dueDateFrom.map(d => i.dueDate > d) &&? + q.dueDateTo.map(d => i.dueDate < d) &&? + q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? + q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&? + TagItemName + .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) + .map(subsel => i.id.in(subsel)) &&? + TagItemName + .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) + .map(subsel => i.id.notIn(subsel)) - // exclusive tags are OR-ed - val tagSelectsExcl = - TagItemName.itemsWithTagOrCategory(q.tagsExclude, q.tagCategoryExcl) - - val iFolder = IC.folder.prefix("i") - val name = q.name.map(_.toLowerCase).map(QueryWildcard.apply) - val allNames = q.allNames.map(_.toLowerCase).map(QueryWildcard.apply) - val sourceName = q.source.map(_.toLowerCase).map(QueryWildcard.apply) - val cond = and( - IC.cid.prefix("i").is(q.account.collective), - IC.state.prefix("i").isOneOf(q.states), - IC.incoming.prefix("i").isOrDiscard(q.direction), - name - .map(n => IC.name.prefix("i").lowerLike(n)) - .getOrElse(Fragment.empty), - allNames - .map(n => - or( - org.name.column.lowerLike(n), - pers0.name.column.lowerLike(n), - pers1.name.column.lowerLike(n), - equip.name.oldColumn.prefix("e1").lowerLike(n), - IC.name.prefix("i").lowerLike(n), - IC.notes.prefix("i").lowerLike(n) - ) - ) - .getOrElse(Fragment.empty), - pers0.pid.column.isOrDiscard(q.corrPerson), - org.oid.column.isOrDiscard(q.corrOrg), - pers1.pid.column.isOrDiscard(q.concPerson), - equip.eid.oldColumn.prefix("e1").isOrDiscard(q.concEquip), - f.id.column.isOrDiscard(q.folder), - if (q.tagsInclude.isEmpty && q.tagCategoryIncl.isEmpty) Fragment.empty - else - IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl - .reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", - if (q.tagsExclude.isEmpty && q.tagCategoryExcl.isEmpty) Fragment.empty - else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")", - q.dateFrom - .map(d => - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d" - ) - .getOrElse(Fragment.empty), - q.dateTo - .map(d => - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d" - ) - .getOrElse(Fragment.empty), - q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty), - q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty), - sourceName.map(s => IC.source.prefix("i").lowerLike(s)).getOrElse(Fragment.empty), - q.itemIds - .map(ids => - NonEmptyList - .fromList(ids.toList) - .map(nel => IC.id.prefix("i").isIn(nel)) - .getOrElse(IC.id.prefix("i").is("")) - ) - .getOrElse(Fragment.empty), - or(iFolder.isNull, iFolder.isIn(QFolder.findMemberFolderIds(q.account).build)) - ) - - val order = q.orderAsc match { - case Some(co) => - orderBy(coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) ++ fr"ASC") - case None => - orderBy( - coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"DESC" - ) - } - val limitOffset = - if (batch == Batch.all) Fragment.empty - else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}" - - val query = findItemsBase(q, true, maxNoteLen, Seq.empty) - val frag = - query ++ fr"WHERE" ++ cond ++ order ++ limitOffset - logger.trace(s"List $batch items: $frag") - frag.query[ListItem].stream + val sql = findItemsBase(q, maxNoteLen) + .changeWhere(cond) + .limit(batch) + .build + logger.info(s"List $batch items: $sql") + sql.query[ListItem].stream } case class SelectedItem(itemId: Ident, weight: Double) @@ -503,27 +426,50 @@ object QItem { q: Query, maxNoteLen: Int, items: Set[SelectedItem] - ): Stream[ConnectionIO, ListItem] = + ): Stream[ConnectionIO, ListItem] = { + import docspell.store.qb.DSL._ + if (items.isEmpty) Stream.empty else { - val IC = RItem.Columns - val values = items - .map(it => fr"(${it.itemId}, ${it.weight})") - .reduce((r, e) => r ++ fr"," ++ e) + val i = RItem.as("i") - val from = findItemsBase( - q, - true, - maxNoteLen, - Seq(fr"tids.weight"), - ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") - ) ++ - fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++ - fr"ORDER BY tids.weight DESC" + object Tids extends TableDef { + val tableName = "tids" + val alias: Option[String] = Some("tw") + val itemId = Column[Ident]("item_id", this) + val weight = Column[Double]("weight", this) + val all = Vector[Column[_]](itemId, weight) + } - logger.trace(s"fts query: $from") + val cte = + CteBind( + Tids, + Tids.all, + Select.RawSelect( + fr"VALUES" ++ + items + .map(it => fr"(${it.itemId}, ${it.weight})") + .reduce((r, e) => r ++ fr"," ++ e) + ) + ) + + val from = findItemsBase(q, maxNoteLen) + .appendCte(cte) + .appendSelect(Tids.weight.s) + .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id)) + .orderBy(Tids.weight.desc) + .build + +// Seq(fr"tids.weight"), +// ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") +// ) ++ +// fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++ +// fr"ORDER BY tids.weight DESC" + + logger.info(s"fts query: $from") from.query[ListItem].stream } + } case class AttachmentLight( id: Ident, @@ -581,7 +527,6 @@ object QItem { } private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val a = RAttachment.as("a") @@ -605,10 +550,9 @@ object QItem { } yield tn + rn + n + mn + cf private def findByFileIdsQuery( - fileMetaIds: NonEmptyList[Ident], - states: Option[NonEmptyList[ItemState]] + fileMetaIds: Nel[Ident], + states: Option[Nel[ItemState]] ): Select.SimpleSelect = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val i = RItem.as("i") @@ -629,7 +573,7 @@ object QItem { } def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] = - NonEmptyList.fromList(fileMetaIds.toList) match { + Nel.fromList(fileMetaIds.toList) match { case Some(nel) => findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option case None => @@ -638,9 +582,9 @@ object QItem { def findByFileIds( fileMetaIds: Seq[Ident], - states: NonEmptyList[ItemState] + states: Nel[ItemState] ): ConnectionIO[Vector[RItem]] = - NonEmptyList.fromList(fileMetaIds.toList) match { + Nel.fromList(fileMetaIds.toList) match { case Some(nel) => findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector] case None => @@ -648,7 +592,6 @@ object QItem { } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val m1 = RFileMeta.as("m1") @@ -704,7 +647,6 @@ object QItem { collective: Ident, chunkSize: Int ): Stream[ConnectionIO, Ident] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val i = RItem.as("i") @@ -724,7 +666,6 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - import docspell.store.qb._ import docspell.store.qb.DSL._ val tag = RTag.as("t") diff --git a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala index 4416de08..5ab70d20 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala @@ -2,6 +2,9 @@ package docspell.store.queries object QueryWildcard { + def lower(s: String): String = + apply(s.toLowerCase) + def apply(value: String): String = { def prefix(n: String) = if (n.startsWith("*")) s"%${n.substring(1)}" diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index 26372748..e6cd356c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -6,7 +6,6 @@ import fs2.Stream import docspell.common._ import docspell.store.qb.DSL._ -import docspell.store.qb.TableDef import docspell.store.qb._ import bitpeace.FileMeta @@ -115,16 +114,6 @@ object RAttachment { def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = { import bitpeace.sql._ -// val cols = RFileMeta.Columns.all.map(_.prefix("m")) -// val aId = id.prefix("a") -// val aFileMeta = fileId.prefix("a") -// val mId = RFileMeta.Columns.id.prefix("m") -// -// val from = -// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId) -// val cond = aId.is(attachId) -// -// selectSimple(cols, from, cond).query[FileMeta].option val m = RFileMeta.as("m") val a = RAttachment.as("a") Select( @@ -167,14 +156,6 @@ object RAttachment { attachId: Ident, collective: Ident ): ConnectionIO[Boolean] = { -// val aId = id.prefix("a") -// val aItem = itemId.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val from = -// table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) -// val cond = and(iColl.is(collective), aId.is(attachId)) -// selectCount(id, from, cond).query[Int].unique.map(_ > 0) val a = RAttachment.as("a") val i = RItem.as("i") Select( @@ -189,12 +170,6 @@ object RAttachment { id: Ident, coll: Ident ): ConnectionIO[Vector[RAttachment]] = { -// val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id -// .prefix("i") -// .is(itemId.prefix("a")) ++ -// fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) -// q.query[RAttachment].to[Vector] val a = RAttachment.as("a") val i = RItem.as("i") Select( @@ -210,29 +185,6 @@ object RAttachment { coll: Ident, fileIds: NonEmptyList[Ident] ): ConnectionIO[Vector[RAttachment]] = { - -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val aItem = Columns.itemId.prefix("a") -// val aId = Columns.id.prefix("a") -// val aFile = Columns.fileId.prefix("a") -// val sId = RAttachmentSource.Columns.id.prefix("s") -// val sFile = RAttachmentSource.Columns.fileId.prefix("s") -// val rId = RAttachmentArchive.Columns.id.prefix("r") -// val rFile = RAttachmentArchive.Columns.fileId.prefix("r") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"LEFT JOIN" ++ -// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"LEFT JOIN" ++ -// RAttachmentArchive.table ++ fr"r ON" ++ rId.is(aId) -// -// val cond = and( -// iId.is(id), -// iColl.is(coll), -// or(aFile.isIn(fileIds), sFile.isIn(fileIds), rFile.isIn(fileIds)) -// ) -// -// selectSimple(all.map(_.prefix("a")), from, cond).query[RAttachment].to[Vector] val i = RItem.as("i") val a = RAttachment.as("a") val s = RAttachmentSource.as("s") @@ -255,19 +207,6 @@ object RAttachment { ): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ -// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) -// val afileMeta = fileId.prefix("a") -// val aItem = itemId.prefix("a") -// val mId = RFileMeta.Columns.id.prefix("m") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = -// table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ aItem.is(iId) -// val cond = Seq(aItem.is(id), iColl.is(coll)) -// -// selectSimple(cols, from, and(cond)).query[(RAttachment, FileMeta)].to[Vector] val a = RAttachment.as("a") val m = RFileMeta.as("m") val i = RItem.as("i") @@ -283,9 +222,6 @@ object RAttachment { def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { import bitpeace.sql._ -// val q = -// fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filzemeta m -// WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" val a = RAttachment.as("a") val m = RFileMeta.as("m") Select( @@ -313,24 +249,6 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aItem = Columns.itemId.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val cols = all.map(_.prefix("a")) -// -// coll match { -// case Some(cid) => -// val join = table ++ fr"a INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) -// val cond = iColl.is(cid) -// selectSimple(cols, join, cond) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// case None => -// selectSimple(cols, table, Fragment.empty) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// } val a = RAttachment.as("a") val i = RItem.as("i") @@ -350,19 +268,6 @@ object RAttachment { } def findAllWithoutPageCount(chunkSize: Int): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aCreated = Columns.created.prefix("a") -// val mId = RAttachmentMeta.Columns.id.prefix("m") -// val mPages = RAttachmentMeta.Columns.pages.prefix("m") -// -// val cols = all.map(_.prefix("a")) -// val join = table ++ fr"a LEFT OUTER JOIN" ++ -// RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) -// val cond = mPages.isNull -// -// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) val a = RAttachment.as("a") val m = RAttachmentMeta.as("m") Select( @@ -377,33 +282,6 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aItem = Columns.itemId.prefix("a") -// val aCreated = Columns.created.prefix("a") -// val pId = RAttachmentPreview.Columns.id.prefix("p") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val cols = all.map(_.prefix("a")) -// val baseJoin = -// table ++ fr"a LEFT OUTER JOIN" ++ -// RAttachmentPreview.table ++ fr"p ON" ++ pId.is(aId) -// -// val baseCond = -// Seq(pId.isNull) -// -// coll match { -// case Some(cid) => -// val join = baseJoin ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) -// val cond = and(baseCond ++ Seq(iColl.is(cid))) -// (selectSimple(cols, join, cond) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// case None => -// (selectSimple(cols, baseJoin, and(baseCond)) ++ orderBy(aCreated.desc)) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) -// } val a = RAttachment.as("a") val p = RAttachmentPreview.as("p") val i = RItem.as("i") @@ -420,32 +298,11 @@ object RAttachment { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, RAttachment] = { -// val aId = Columns.id.prefix("a") -// val aItem = Columns.itemId.prefix("a") -// val aFile = Columns.fileId.prefix("a") -// val sId = RAttachmentSource.Columns.id.prefix("s") -// val sFile = RAttachmentSource.Columns.fileId.prefix("s") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// val mId = RFileMeta.Columns.id.prefix("m") -// val mType = RFileMeta.Columns.mimetype.prefix("m") val pdfType = "application/pdf%" -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachmentSource.table ++ fr"s ON" ++ sId.is(aId) ++ fr"INNER JOIN" ++ -// RItem.table ++ fr"i ON" ++ iId.is(aItem) ++ fr"INNER JOIN" ++ -// RFileMeta.table ++ fr"m ON" ++ aFile.is(mId) -// val where = coll match { -// case Some(cid) => and(iColl.is(cid), aFile.is(sFile), mType.lowerLike(pdfType)) -// case None => and(aFile.is(sFile), mType.lowerLike(pdfType)) -// } -// selectSimple(all.map(_.prefix("a")), from, where) -// .query[RAttachment] -// .streamWithChunkSize(chunkSize) - val a = RAttachment.as("a") - val s = RAttachmentSource.as("s") - val i = RItem.as("i") - val m = RFileMeta.as("m") + val a = RAttachment.as("a") + val s = RAttachmentSource.as("s") + val i = RItem.as("i") + val m = RFileMeta.as("m") Select( select(a.all), diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index 5e9f4eb2..cca55b8f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -21,29 +21,29 @@ object RTagItem { val tagId = Column[Ident]("tid", this) val all = NonEmptyList.of[Column[_]](tagItemId, itemId, tagId) } - val t = Table(None) + val T = Table(None) def as(alias: String): Table = Table(Some(alias)) def insert(v: RTagItem): ConnectionIO[Int] = - DML.insert(t, t.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") + DML.insert(T, T.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") def deleteItemTags(item: Ident): ConnectionIO[Int] = - DML.delete(t, t.itemId === item) + DML.delete(T, T.itemId === item) def deleteItemTags(items: NonEmptyList[Ident], cid: Ident): ConnectionIO[Int] = - DML.delete(t, t.itemId.in(RItem.filterItemsFragment(items, cid))) + DML.delete(T, T.itemId.in(RItem.filterItemsFragment(items, cid))) def deleteTag(tid: Ident): ConnectionIO[Int] = - DML.delete(t, t.tagId === tid) + DML.delete(T, T.tagId === tid) def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] = - run(select(t.all), from(t), t.itemId === item).query[RTagItem].to[Vector] + run(select(T.all), from(T), T.itemId === item).query[RTagItem].to[Vector] def findAllIn(item: Ident, tags: Seq[Ident]): ConnectionIO[Vector[RTagItem]] = NonEmptyList.fromList(tags.toList) match { case Some(nel) => - run(select(t.all), from(t), t.itemId === item && t.tagId.in(nel)) + run(select(T.all), from(T), T.itemId === item && T.tagId.in(nel)) .query[RTagItem] .to[Vector] case None => @@ -55,7 +55,7 @@ object RTagItem { case None => 0.pure[ConnectionIO] case Some(nel) => - DML.delete(t, t.itemId === item && t.tagId.in(nel)) + DML.delete(T, T.itemId === item && T.tagId.in(nel)) } def setAllTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = @@ -67,8 +67,8 @@ object RTagItem { ) n <- DML .insertMany( - t, - t.all, + T, + T.all, entities.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") ) } yield n diff --git a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala index 45ef618a..9b7d8b77 100644 --- a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala +++ b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala @@ -4,8 +4,7 @@ import cats.data.NonEmptyList import docspell.common._ import docspell.store.qb.DSL._ - -import doobie._ +import docspell.store.qb.{Condition, Select} /** A helper class combining information from `RTag` and `RTagItem`. * This is not a "record", there is no corresponding table. @@ -20,32 +19,80 @@ case class TagItemName( ) object TagItemName { + private val ti = RTagItem.as("ti") + private val t = RTag.as("t") - def itemsInCategory(cats: NonEmptyList[String]): Fragment = { - val catsLower = cats.map(_.toLowerCase) - val ti = RTagItem.as("ti") - val t = RTag.as("t") - val join = from(t).innerJoin(ti, t.tid === ti.tagId) - if (cats.tail.isEmpty) - run(select(ti.itemId), join, t.category.likes(catsLower.head)) - else - run(select(ti.itemId), join, t.category.inLower(cats)) - } + private val taggedItems = + from(t).innerJoin(ti, t.tid === ti.tagId) - def itemsWithTagOrCategory(tags: List[Ident], cats: List[String]): Fragment = { + private def orTags(tags: NonEmptyList[Ident]): Condition = + ti.tagId.in(tags) + + private def orCategory(cats: NonEmptyList[String]): Condition = + t.category.inLower(cats) + + def itemsInEitherCategory(cats: NonEmptyList[String]): Select = + Select(ti.itemId.s, taggedItems, orCategory(cats)).distinct + + def itemsInAllCategories(cats: NonEmptyList[String]): Select = + intersect( + cats.map(cat => Select(ti.itemId.s, taggedItems, t.category === cat).distinct) + ) + + def itemsWithEitherTag(tags: NonEmptyList[Ident]): Select = + Select(ti.itemId.s, from(ti), orTags(tags)).distinct + + def itemsWithAllTags(tags: NonEmptyList[Ident]): Select = + intersect(tags.map(tid => Select(ti.itemId.s, from(ti), ti.tagId === tid).distinct)) + + def itemsWithEitherTagOrCategory( + tags: NonEmptyList[Ident], + cats: NonEmptyList[String] + ): Select = + Select(ti.itemId.s, taggedItems, orTags(tags) || orCategory(cats)) + + def itemsWithAllTagAndCategory( + tags: NonEmptyList[Ident], + cats: NonEmptyList[String] + ): Select = + Select( + ti.itemId.s, + from(ti), + ti.itemId.in(itemsWithAllTags(tags)) && + ti.itemId.in(itemsInAllCategories(cats)) + ) + + def itemsWithEitherTagOrCategory( + tags: List[Ident], + cats: List[String] + ): Option[Select] = { val catsLower = cats.map(_.toLowerCase) - val ti = RTagItem.as("ti") - val t = RTag.as("t") - val join = from(t).innerJoin(ti, t.tid === ti.tagId) (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match { case (Some(tagNel), Some(catNel)) => - run(select(ti.itemId), join, t.tid.in(tagNel) || t.category.inLower(catNel)) + Some(itemsWithEitherTagOrCategory(tagNel, catNel)) case (Some(tagNel), None) => - run(select(ti.itemId), join, t.tid.in(tagNel)) + Some(itemsWithEitherTag(tagNel)) case (None, Some(catNel)) => - run(select(ti.itemId), join, t.category.inLower(catNel)) + Some(itemsInEitherCategory(catNel)) case (None, None) => - Fragment.empty + None + } + } + + def itemsWithAllTagAndCategory( + tags: List[Ident], + cats: List[String] + ): Option[Select] = { + val catsLower = cats.map(_.toLowerCase) + (NonEmptyList.fromList(tags), NonEmptyList.fromList(catsLower)) match { + case (Some(tagNel), Some(catNel)) => + Some(itemsWithAllTagAndCategory(tagNel, catNel)) + case (Some(tagNel), None) => + Some(itemsWithAllTags(tagNel)) + case (None, Some(catNel)) => + Some(itemsInAllCategories(catNel)) + case (None, None) => + None } } } diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala index e35c73d9..7d5dc64d 100644 --- a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -48,14 +48,14 @@ object QueryBuilderTest extends SimpleTestSuite { assertEquals(f, FromExpr.From(c)) assertEquals(2, joins.size) joins.head match { - case Join.InnerJoin(tbl, cond) => + case FromExpr.Join.InnerJoin(FromExpr.Relation.Table(tbl), cond) => assertEquals(tbl, owner) assertEquals(cond, c.ownerId === owner.id) case _ => fail("Unexpected join result") } joins.tail.head match { - case Join.LeftJoin(tbl, cond) => + case FromExpr.Join.LeftJoin(FromExpr.Relation.Table(tbl), cond) => assertEquals(tbl, lecturer) assertEquals(cond, c.lecturerId === lecturer.id) case _ => diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala new file mode 100644 index 00000000..4a3f062e --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/impl/DSLTest.scala @@ -0,0 +1,19 @@ +package docspell.store.qb.impl + +import minitest._ +import docspell.store.qb._ +import docspell.store.qb.DSL._ +import docspell.store.qb.model.{CourseRecord, PersonRecord} + +object DSLTest extends SimpleTestSuite { + + val course = CourseRecord.as("c") + val person = PersonRecord.as("p") + + test("and") { + val c = course.lessons > 4 && person.id === 3 && person.name.like("%a%") + val expect = + Condition.And(course.lessons > 4, person.id === 3, person.name.like("%a%")) + assertEquals(c, expect) + } +} From d1606d6f16033c5c3eee8db5671bd8ecc856d1b3 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 13:41:26 +0100 Subject: [PATCH 19/38] Remove old commented code --- .../store/records/RAttachmentArchive.scala | 49 ---------------- .../store/records/RAttachmentPreview.scala | 57 ------------------- .../store/records/RAttachmentSource.scala | 51 ----------------- 3 files changed, 157 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala index ab97b9b2..f32c06fd 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala @@ -74,19 +74,6 @@ object RAttachmentArchive { attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentArchive]] = { -// val bId = RAttachment.Columns.id.prefix("b") -// val aId = Columns.id.prefix("a") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) -// -// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) -// -// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].option val b = RAttachment.as("b") val a = RAttachmentArchive.as("a") val i = RItem.as("i") @@ -104,20 +91,6 @@ object RAttachmentArchive { messageIds: NonEmptyList[String], collective: Ident ): ConnectionIO[Vector[RAttachmentArchive]] = { -// val bId = RAttachment.Columns.id.prefix("b") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val aMsgId = Columns.messageId.prefix("a") -// val aId = Columns.id.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) -// -// val where = and(aMsgId.isIn(messageIds), iColl.is(collective)) -// -// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].to[Vector] val b = RAttachment.as("b") val a = RAttachmentArchive.as("a") val i = RItem.as("i") @@ -135,24 +108,6 @@ object RAttachmentArchive { ): ConnectionIO[Vector[(RAttachmentArchive, FileMeta)]] = { import bitpeace.sql._ -// val aId = Columns.id.prefix("a") -// val afileMeta = fileId.prefix("a") -// val bPos = RAttachment.Columns.position.prefix("b") -// val bId = RAttachment.Columns.id.prefix("b") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val mId = RFileMeta.as("m").id.column -// -// val cols = all.map(_.prefix("a")) ++ RFileMeta.as("m").all.map(_.column).toList -// val from = table ++ fr"a INNER JOIN" ++ -// Fragment.const(RFileMeta.T.tableName) ++ fr"m ON" ++ afileMeta.is( -// mId -// ) ++ fr"INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) -// val where = bItem.is(id) -// -// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) -// .query[(RAttachmentArchive, FileMeta)] -// .to[Vector] val a = RAttachmentArchive.as("a") val b = RAttachment.as("b") val m = RFileMeta.as("m") @@ -170,14 +125,10 @@ object RAttachmentArchive { * no archive for the given attachment. */ def countEntries(attachId: Ident): ConnectionIO[Int] = -// val qFileId = selectSimple(Seq(fileId), table, id.is(attachId)) -// val q = selectCount(id, table, fileId.isSubquery(qFileId)) -// q.query[Int].unique Select( count(T.id).s, from(T), T.fileId.in(Select(T.fileId.s, from(T), T.id === attachId)) ).build.query[Int].unique - //TODO this looks strange, can be simplified } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index 290efd50..24f6fc80 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -60,19 +60,6 @@ object RAttachmentPreview { attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentPreview]] = { -// val bId = RAttachment.Columns.id.prefix("b") -// val aId = Columns.id.prefix("a") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) -// -// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) -// -// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentPreview].option val b = RAttachment.as("b") val a = RAttachmentPreview.as("a") val i = RItem.as("i") @@ -87,14 +74,6 @@ object RAttachmentPreview { } def findByItem(itemId: Ident): ConnectionIO[Vector[RAttachmentPreview]] = { -// val sId = Columns.id.prefix("s") -// val aId = RAttachment.Columns.id.prefix("a") -// val aItem = RAttachment.Columns.itemId.prefix("a") -// -// val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) -// selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) -// .query[RAttachmentPreview] -// .to[Vector] val s = RAttachmentPreview.as("s") val a = RAttachment.as("a") Select( @@ -109,25 +88,6 @@ object RAttachmentPreview { itemId: Ident, coll: Ident ): ConnectionIO[Option[RAttachmentPreview]] = { -// val sId = Columns.id.prefix("s") -// val aId = RAttachment.Columns.id.prefix("a") -// val aItem = RAttachment.Columns.itemId.prefix("a") -// val aPos = RAttachment.Columns.position.prefix("a") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = -// table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ iId.is(aItem) -// -// selectSimple( -// all.map(_.prefix("s")) ++ List(aPos), -// from, -// and(aItem.is(itemId), iColl.is(coll)) -// ) -// .query[(RAttachmentPreview, Int)] -// .to[Vector] -// .map(_.sortBy(_._2).headOption.map(_._1)) val s = RAttachmentPreview.as("s") val a = RAttachment.as("a") val i = RItem.as("i") @@ -149,22 +109,6 @@ object RAttachmentPreview { ): ConnectionIO[Vector[(RAttachmentPreview, FileMeta)]] = { import bitpeace.sql._ -// val aId = Columns.id.prefix("a") -// val afileMeta = fileId.prefix("a") -// val bPos = RAttachment.Columns.position.prefix("b") -// val bId = RAttachment.Columns.id.prefix("b") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val mId = RFileMeta.Columns.id.prefix("m") -// -// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) -// val from = table ++ fr"a INNER JOIN" ++ -// RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) -// val where = bItem.is(id) -// -// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) -// .query[(RAttachmentPreview, FileMeta)] -// .to[Vector] val a = RAttachmentPreview.as("a") val b = RAttachment.as("b") val m = RFileMeta.as("m") @@ -177,5 +121,4 @@ object RAttachmentPreview { b.itemId === id ).orderBy(b.position.asc).build.query[(RAttachmentPreview, FileMeta)].to[Vector] } - } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala index 4d94743b..20629645 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala @@ -70,19 +70,6 @@ object RAttachmentSource { from(s).innerJoin(a, a.id === s.id), a.id === attachId && a.fileId <> s.fileId ).build.query[Int].unique.map(_ > 0) - -// val sId = Columns.id.prefix("s") -// val sFile = Columns.fileId.prefix("s") -// val aId = RAttachment.Columns.id.prefix("a") -// val aFile = RAttachment.Columns.fileId.prefix("a") -// -// val from = table ++ fr"s INNER JOIN" ++ -// RAttachment.table ++ fr"a ON" ++ aId.is(sId) -// -// selectCount(aId, from, and(aId.is(attachId), aFile.isNot(sFile))) -// .query[Int] -// .unique -// .map(_ > 0) } def delete(attachId: Ident): ConnectionIO[Int] = @@ -92,19 +79,6 @@ object RAttachmentSource { attachId: Ident, collective: Ident ): ConnectionIO[Option[RAttachmentSource]] = { -// val bId = RAttachment.Columns.id.prefix("b") -// val aId = Columns.id.prefix("a") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val iId = RItem.Columns.id.prefix("i") -// val iColl = RItem.Columns.cid.prefix("i") -// -// val from = table ++ fr"a INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++ -// fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId) -// -// val where = and(aId.is(attachId), bId.is(attachId), iColl.is(collective)) -// -// selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentSource].option val b = RAttachment.as("b") val a = RAttachmentSource.as("a") val i = RItem.as("i") @@ -119,15 +93,6 @@ object RAttachmentSource { } def findByItem(itemId: Ident): ConnectionIO[Vector[RAttachmentSource]] = { -// val sId = Columns.id.prefix("s") -// val aId = RAttachment.Columns.id.prefix("a") -// val aItem = RAttachment.Columns.itemId.prefix("a") -// -// val from = table ++ fr"s INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ sId.is(aId) -// selectSimple(all.map(_.prefix("s")), from, aItem.is(itemId)) -// .query[RAttachmentSource] -// .to[Vector] - val s = RAttachmentSource.as("s") val a = RAttachment.as("a") Select(select(s.all), from(s).innerJoin(a, a.id === s.id), a.itemId === itemId).build @@ -140,22 +105,6 @@ object RAttachmentSource { ): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = { import bitpeace.sql._ -// val aId = Columns.id.prefix("a") -// val afileMeta = fileId.prefix("a") -// val bPos = RAttachment.Columns.position.prefix("b") -// val bId = RAttachment.Columns.id.prefix("b") -// val bItem = RAttachment.Columns.itemId.prefix("b") -// val mId = RFileMeta.Columns.id.prefix("m") -// -// val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) -// val from = table ++ fr"a INNER JOIN" ++ -// RFileMeta.table ++ fr"m ON" ++ afileMeta.is(mId) ++ fr"INNER JOIN" ++ -// RAttachment.table ++ fr"b ON" ++ aId.is(bId) -// val where = bItem.is(id) -// -// (selectSimple(cols, from, where) ++ orderBy(bPos.asc)) -// .query[(RAttachmentSource, FileMeta)] -// .to[Vector] val a = RAttachmentSource.as("a") val b = RAttachment.as("b") val m = RFileMeta.as("m") From 2cecd018379c3261c8dae5f610306d63ed26168c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 17:22:25 +0100 Subject: [PATCH 20/38] Convert rest of QItem --- .../scala/docspell/store/queries/QItem.scala | 91 ++++--------------- 1 file changed, 20 insertions(+), 71 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index cea2177e..c4f1884e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -10,7 +10,6 @@ import fs2.Stream import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store -import docspell.store.impl.DoobieMeta._ import docspell.store.qb._ import docspell.store.records._ @@ -21,6 +20,7 @@ import org.log4s._ object QItem { private[this] val logger = getLogger + import docspell.store.qb.DSL._ def moveAttachmentBefore( itemId: Ident, @@ -87,57 +87,30 @@ object QItem { } def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { - import docspell.store.impl.Implicits._ - val equip = REquipment.as("e") val org = ROrganization.as("o") val pers0 = RPerson.as("p0") val pers1 = RPerson.as("p1") val f = RFolder.as("f") - - val IC = RItem.Columns.all.map(_.prefix("i")) - val OC = org.all.map(_.column).toList - val P0C = pers0.all.map(_.column).toList - val P1C = pers1.all.map(_.column).toList - val EC = equip.all.map(_.oldColumn).map(_.prefix("e")).toList - val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) - val FC = List(f.id.column, f.name.column) + val i = RItem.as("i") + val ref = RItem.as("ref") val cq = - selectSimple( - IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC ++ FC, - RItem.table ++ fr"i", - Fragment.empty - ) ++ - fr"LEFT JOIN" ++ Fragment.const( - org.tableName - ) ++ fr"o ON" ++ RItem.Columns.corrOrg - .prefix("i") - .is(org.oid.column) ++ - fr"LEFT JOIN" ++ Fragment.const( - pers0.tableName - ) ++ fr"p0 ON" ++ RItem.Columns.corrPerson - .prefix("i") - .is(pers0.pid.column) ++ - fr"LEFT JOIN" ++ Fragment.const( - pers1.tableName - ) ++ fr"p1 ON" ++ RItem.Columns.concPerson - .prefix("i") - .is(pers1.pid.column) ++ - fr"LEFT JOIN" ++ Fragment.const( - equip.tableName - ) ++ fr"e ON" ++ RItem.Columns.concEquipment - .prefix("i") - .is(equip.eid.oldColumn.prefix("e")) ++ - fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo - .prefix("i") - .is(RItem.Columns.id.prefix("ref")) ++ - fr"LEFT JOIN" ++ Fragment.const( - RFolder.T.tableName - ) ++ fr"f ON" ++ RItem.Columns.folder - .prefix("i") - .is(f.id.column) ++ - fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) + Select( + select(i.all, org.all, pers0.all, pers1.all, equip.all) + .append(ref.id.s) + .append(ref.name.s) + .append(f.id.s) + .append(f.name.s), + from(i) + .leftJoin(org, org.oid === i.corrOrg) + .leftJoin(pers0, pers0.pid === i.corrPerson) + .leftJoin(pers1, pers1.pid === i.concPerson) + .leftJoin(equip, equip.eid === i.concEquipment) + .leftJoin(ref, ref.id === i.inReplyTo) + .leftJoin(f, f.id === i.folder), + i.id === id + ).build val q = cq .query[ @@ -152,7 +125,7 @@ object QItem { ) ] .option - logger.trace(s"Find item query: $cq") + logger.info(s"Find item query: $cq") val attachs = RAttachment.findByItemWithMeta(id) val sources = RAttachmentSource.findByItemWithMeta(id) val archives = RAttachmentArchive.findByItemWithMeta(id) @@ -174,8 +147,6 @@ object QItem { def findCustomFieldValuesForItem( itemId: Ident ): ConnectionIO[Vector[ItemFieldValue]] = { - import docspell.store.qb.DSL._ - val cf = RCustomField.as("cf") val cv = RCustomFieldValue.as("cvf") @@ -264,8 +235,6 @@ object QItem { coll: Ident, values: Seq[CustomValue] ): Option[Select] = { - import docspell.store.qb.DSL._ - val cf = RCustomField.as("cf") val cv = RCustomFieldValue.as("cv") @@ -286,8 +255,6 @@ object QItem { } private def findItemsBase(q: Query, noteMaxLen: Int): Select = { - import docspell.store.qb.DSL._ - object Attachs extends TableDef { val tableName = "attachs" val aliasName = "cta" @@ -371,8 +338,6 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - import docspell.store.qb.DSL._ - val equip = REquipment.as("e1") val org = ROrganization.as("o0") val pers0 = RPerson.as("p0") @@ -426,9 +391,7 @@ object QItem { q: Query, maxNoteLen: Int, items: Set[SelectedItem] - ): Stream[ConnectionIO, ListItem] = { - import docspell.store.qb.DSL._ - + ): Stream[ConnectionIO, ListItem] = if (items.isEmpty) Stream.empty else { val i = RItem.as("i") @@ -469,7 +432,6 @@ object QItem { logger.info(s"fts query: $from") from.query[ListItem].stream } - } case class AttachmentLight( id: Ident, @@ -527,8 +489,6 @@ object QItem { } private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - import docspell.store.qb.DSL._ - val a = RAttachment.as("a") val m = RAttachmentMeta.as("m") @@ -553,8 +513,6 @@ object QItem { fileMetaIds: Nel[Ident], states: Option[Nel[ItemState]] ): Select.SimpleSelect = { - import docspell.store.qb.DSL._ - val i = RItem.as("i") val a = RAttachment.as("a") val s = RAttachmentSource.as("s") @@ -592,8 +550,6 @@ object QItem { } def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = { - import docspell.store.qb.DSL._ - val m1 = RFileMeta.as("m1") val m2 = RFileMeta.as("m2") val m3 = RFileMeta.as("m3") @@ -629,9 +585,6 @@ object QItem { coll: Option[Ident], chunkSize: Int ): Stream[ConnectionIO, NameAndNotes] = { - import docspell.store.qb._ - import docspell.store.qb.DSL._ - val i = RItem.as("i") Select( @@ -647,8 +600,6 @@ object QItem { collective: Ident, chunkSize: Int ): Stream[ConnectionIO, Ident] = { - import docspell.store.qb.DSL._ - val i = RItem.as("i") Select(i.id.s, from(i), i.cid === collective && i.state === ItemState.confirmed) .orderBy(i.created.desc) @@ -666,8 +617,6 @@ object QItem { tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - import docspell.store.qb.DSL._ - val tag = RTag.as("t") val a = RAttachment.as("a") val am = RAttachmentMeta.as("m") From 278b1c22c9953239da346e9db49b22c920f4b88b Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 19:18:14 +0100 Subject: [PATCH 21/38] Remove old code --- .../scala/docspell/store/impl/Column.scala | 139 ------------------ .../docspell/store/impl/DoobieSyntax.scala | 103 ------------- .../scala/docspell/store/impl/Implicits.scala | 15 -- .../scala/docspell/store/queries/QItem.scala | 14 +- .../docspell/store/records/RAttachment.scala | 14 -- .../store/records/RAttachmentArchive.scala | 13 -- .../store/records/RAttachmentMeta.scala | 12 -- .../store/records/RAttachmentPreview.scala | 11 -- .../store/records/RAttachmentSource.scala | 11 -- .../scala/docspell/store/records/RItem.scala | 42 ------ 10 files changed, 4 insertions(+), 370 deletions(-) delete mode 100644 modules/store/src/main/scala/docspell/store/impl/Column.scala delete mode 100644 modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala delete mode 100644 modules/store/src/main/scala/docspell/store/impl/Implicits.scala diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala deleted file mode 100644 index 4dec4d6c..00000000 --- a/modules/store/src/main/scala/docspell/store/impl/Column.scala +++ /dev/null @@ -1,139 +0,0 @@ -package docspell.store.impl - -import cats.data.NonEmptyList - -import docspell.store.impl.DoobieSyntax._ - -import doobie._ -import doobie.implicits._ - -case class Column(name: String, ns: String = "", alias: String = "") { - - val f = { - val col = - if (ns.isEmpty) Fragment.const(name) - else Fragment.const(ns + "." + name) - if (alias.isEmpty) col - else col ++ fr"as" ++ Fragment.const(alias) - } - - def lowerLike[A: Put](value: A): Fragment = - fr"lower(" ++ f ++ fr") LIKE $value" - - def like[A: Put](value: A): Fragment = - f ++ fr"LIKE $value" - - def is[A: Put](value: A): Fragment = - f ++ fr" = $value" - - def lowerIs[A: Put](value: A): Fragment = - fr"lower(" ++ f ++ fr") = $value" - - def is[A: Put](ov: Option[A]): Fragment = - ov match { - case Some(v) => f ++ fr" = $v" - case None => f ++ fr"is null" - } - - def is(c: Column): Fragment = - f ++ fr"=" ++ c.f - - def isSubquery(sq: Fragment): Fragment = - f ++ fr"=" ++ fr"(" ++ sq ++ fr")" - - def isNot[A: Put](value: A): Fragment = - f ++ fr"<> $value" - - def isNot(c: Column): Fragment = - f ++ fr"<>" ++ c.f - - def isNull: Fragment = - f ++ fr"is null" - - def isNotNull: Fragment = - f ++ fr"is not null" - - def isIn(values: Seq[Fragment]): Fragment = - f ++ fr"IN (" ++ commas(values) ++ fr")" - - def isIn[A: Put](values: NonEmptyList[A]): Fragment = - values.tail match { - case Nil => - is(values.head) - case _ => - isIn(values.map(a => sql"$a").toList) - } - - def isLowerIn[A: Put](values: NonEmptyList[A]): Fragment = - fr"lower(" ++ f ++ fr") IN (" ++ commas(values.map(a => sql"$a").toList) ++ fr")" - - def isIn(frag: Fragment): Fragment = - f ++ fr"IN (" ++ frag ++ fr")" - - def isOrDiscard[A: Put](value: Option[A]): Fragment = - value match { - case Some(v) => is(v) - case None => Fragment.empty - } - - def isOneOf[A: Put](values: Seq[A]): Fragment = { - val vals = values.map(v => sql"$v") - isIn(vals) - } - - def isNotOneOf[A: Put](values: Seq[A]): Fragment = { - val vals = values.map(v => sql"$v") - sql"(" ++ f ++ fr"is null or" ++ f ++ fr"not IN (" ++ commas(vals) ++ sql"))" - } - - def isGt[A: Put](a: A): Fragment = - f ++ fr"> $a" - - def isGte[A: Put](a: A): Fragment = - f ++ fr">= $a" - - def isGt(c: Column): Fragment = - f ++ fr">" ++ c.f - - def isLt[A: Put](a: A): Fragment = - f ++ fr"< $a" - - def isLte[A: Put](a: A): Fragment = - f ++ fr"<= $a" - - def isLt(c: Column): Fragment = - f ++ fr"<" ++ c.f - - def setTo[A: Put](value: A): Fragment = - is(value) - - def setTo[A: Put](va: Option[A]): Fragment = - f ++ fr" = $va" - - def ++(next: Fragment): Fragment = - f.++(next) - - def prefix(ns: String): Column = - Column(name, ns) - - def as(alias: String): Column = - Column(name, ns, alias) - - def desc: Fragment = - f ++ fr"desc" - def asc: Fragment = - f ++ fr"asc" - - def max: Fragment = - fr"MAX(" ++ f ++ fr")" - - def increment[A: Put](a: A): Fragment = - f ++ fr"=" ++ f ++ fr"+ $a" - - def decrement[A: Put](a: A): Fragment = - f ++ fr"=" ++ f ++ fr"- $a" - - def substring(from: Int, many: Int): Fragment = - if (many <= 0 || from < 0) fr"${""}" - else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)" -} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala deleted file mode 100644 index e465d8ea..00000000 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala +++ /dev/null @@ -1,103 +0,0 @@ -package docspell.store.impl - -import cats.data.NonEmptyList - -import docspell.common.Timestamp - -import doobie._ -import doobie.implicits._ - -trait DoobieSyntax { - - def groupBy(c0: Column, cs: Column*): Fragment = - groupBy(NonEmptyList.of(c0, cs: _*)) - - def groupBy(cs: NonEmptyList[Column]): Fragment = - fr" GROUP BY " ++ commas(cs.toList.map(_.f)) - - def coalesce(f0: Fragment, fs: Fragment*): Fragment = - sql" coalesce(" ++ commas(f0 :: fs.toList) ++ sql") " - - def power2(c: Column): Fragment = - sql"power(2," ++ c.f ++ sql")" - - def commas(fs: Seq[Fragment]): Fragment = - fs.reduce(_ ++ Fragment.const(",") ++ _) - - def commas(fa: Fragment, fas: Fragment*): Fragment = - commas(fa :: fas.toList) - - def and(fs: Seq[Fragment]): Fragment = - Fragment.const(" (") ++ fs - .filter(f => !isEmpty(f)) - .reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") - - def and(f0: Fragment, fs: Fragment*): Fragment = - and(f0 :: fs.toList) - - def or(fs: Seq[Fragment]): Fragment = - Fragment.const(" (") ++ fs.reduce(_ ++ Fragment.const(" OR ") ++ _) ++ Fragment.const( - ") " - ) - def or(f0: Fragment, fs: Fragment*): Fragment = - or(f0 :: fs.toList) - - def where(fa: Fragment): Fragment = - if (isEmpty(fa)) Fragment.empty - else Fragment.const(" WHERE ") ++ fa - - def orderBy(fa: Fragment): Fragment = - Fragment.const(" ORDER BY ") ++ fa - - def orderBy(c0: Fragment, cs: Fragment*): Fragment = - fr"ORDER BY" ++ commas(c0 :: cs.toList) - - def updateRow(table: Fragment, where: Fragment, setter: Fragment): Fragment = - Fragment.const("UPDATE ") ++ table ++ Fragment.const(" SET ") ++ setter ++ this.where( - where - ) - - def insertRow(table: Fragment, cols: List[Column], vals: Fragment): Fragment = - Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ - commas(cols.map(_.f)) ++ Fragment.const(") VALUES (") ++ vals ++ Fragment.const(")") - - def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment = - Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ - commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas( - vals.map(f => sql"(" ++ f ++ sql")") - ) - - def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = - selectSimple(commas(cols.map(_.f)), table, where) - - def selectSimple(cols: Fragment, table: Fragment, where: Fragment): Fragment = - Fragment.const("SELECT ") ++ cols ++ - Fragment.const(" FROM ") ++ table ++ this.where(where) - - def selectDistinct(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = - Fragment.const("SELECT DISTINCT ") ++ commas(cols.map(_.f)) ++ - Fragment.const(" FROM ") ++ table ++ this.where(where) - - def selectCount(col: Column, table: Fragment, where: Fragment): Fragment = - Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this - .where( - where - ) - - def deleteFrom(table: Fragment, where: Fragment): Fragment = - fr"DELETE FROM" ++ table ++ this.where(where) - - def withCTE(ps: (String, Fragment)*): Fragment = { - val subsel: Seq[Fragment] = - ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")") - fr"WITH" ++ commas(subsel) - } - - def isEmpty(fragment: Fragment): Boolean = - Fragment.empty.toString() == fragment.toString() - - def currentTime: ConnectionIO[Timestamp] = - Timestamp.current[ConnectionIO] -} - -object DoobieSyntax extends DoobieSyntax diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala deleted file mode 100644 index 2047b301..00000000 --- a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala +++ /dev/null @@ -1,15 +0,0 @@ -package docspell.store.impl - -object Implicits extends DoobieMeta with DoobieSyntax { - - implicit final class LegacySyntax(col: docspell.store.qb.Column[_]) { - def oldColumn: Column = - Column(col.name) - - def column: Column = - col.table.alias match { - case Some(p) => oldColumn.prefix(p) - case None => oldColumn - } - } -} diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index c4f1884e..feba4947 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -10,6 +10,7 @@ import fs2.Stream import docspell.common.syntax.all._ import docspell.common.{IdRef, _} import docspell.store.Store +import docspell.store.qb.DSL._ import docspell.store.qb._ import docspell.store.records._ @@ -20,7 +21,6 @@ import org.log4s._ object QItem { private[this] val logger = getLogger - import docspell.store.qb.DSL._ def moveAttachmentBefore( itemId: Ident, @@ -125,7 +125,7 @@ object QItem { ) ] .option - logger.info(s"Find item query: $cq") + logger.trace(s"Find item query: $cq") val attachs = RAttachment.findByItemWithMeta(id) val sources = RAttachmentSource.findByItemWithMeta(id) val archives = RAttachmentArchive.findByItemWithMeta(id) @@ -382,7 +382,7 @@ object QItem { .changeWhere(cond) .limit(batch) .build - logger.info(s"List $batch items: $sql") + logger.trace(s"List $batch items: $sql") sql.query[ListItem].stream } @@ -423,13 +423,7 @@ object QItem { .orderBy(Tids.weight.desc) .build -// Seq(fr"tids.weight"), -// ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") -// ) ++ -// fr"INNER JOIN tids ON" ++ IC.id.prefix("i").f ++ fr" = tids.item_id" ++ -// fr"ORDER BY tids.weight DESC" - - logger.info(s"fts query: $from") + logger.trace(s"fts query: $from") from.query[ListItem].stream } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index e6cd356c..e781eb89 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -38,20 +38,6 @@ object RAttachment { def as(alias: String): Table = Table(Some(alias)) - val table = fr"attachment" - - object Columns { - import docspell.store.impl._ - - val id = Column("attachid") - val itemId = Column("itemid") - val fileId = Column("filemetaid") - val position = Column("position") - val created = Column("created") - val name = Column("name") - val all = List(id, itemId, fileId, position, created, name) - } - def insert(v: RAttachment): ConnectionIO[Int] = DML.insert( T, diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala index f32c06fd..f71116e2 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala @@ -38,19 +38,6 @@ object RAttachmentArchive { def as(alias: String): Table = Table(Some(alias)) - val table = fr"attachment_archive" - object Columns { - import docspell.store.impl._ - - val id = Column("id") - val fileId = Column("file_id") - val name = Column("filename") - val messageId = Column("message_id") - val created = Column("created") - - val all = List(id, fileId, name, messageId, created) - } - def of(ra: RAttachment, mId: Option[String]): RAttachmentArchive = RAttachmentArchive(ra.id, ra.fileId, ra.name, mId, ra.created) diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index ad5558b8..4adfbad7 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -45,18 +45,6 @@ object RAttachmentMeta { def as(alias: String): Table = Table(Some(alias)) - val table = fr"attachmentmeta" - object Columns { - import docspell.store.impl._ - - val id = Column("attachid") - val content = Column("content") - val nerlabels = Column("nerlabels") - val proposals = Column("itemproposals") - val pages = Column("page_count") - val all = List(id, content, nerlabels, proposals, pages) - } - def insert(v: RAttachmentMeta): ConnectionIO[Int] = DML.insert( T, diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index 24f6fc80..61576930 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -36,17 +36,6 @@ object RAttachmentPreview { def as(alias: String): Table = Table(Some(alias)) - val table = fr"attachment_preview" - object Columns { - import docspell.store.impl._ - val id = Column("id") - val fileId = Column("file_id") - val name = Column("filename") - val created = Column("created") - - val all = List(id, fileId, name, created) - } - def insert(v: RAttachmentPreview): ConnectionIO[Int] = DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala index 20629645..62c1b1d8 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala @@ -36,17 +36,6 @@ object RAttachmentSource { def as(alias: String): Table = Table(Some(alias)) - val table = fr"attachment_source" - object Columns { - import docspell.store.impl._ - val id = Column("id") - val fileId = Column("file_id") - val name = Column("filename") - val created = Column("created") - - val all = List(id, fileId, name, created) - } - def of(ra: RAttachment): RAttachmentSource = RAttachmentSource(ra.id, ra.fileId, ra.name, ra.created) diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 929b3528..6a65ef83 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -108,48 +108,6 @@ object RItem { def as(alias: String): Table = Table(Some(alias)) - val table = fr"item" - object Columns { - import docspell.store.impl._ - - val id = Column("itemid") - val cid = Column("cid") - val name = Column("name") - val itemDate = Column("itemdate") - val source = Column("source") - val incoming = Column("incoming") - val state = Column("state") - val corrOrg = Column("corrorg") - val corrPerson = Column("corrperson") - val concPerson = Column("concperson") - val concEquipment = Column("concequipment") - val inReplyTo = Column("inreplyto") - val dueDate = Column("duedate") - val created = Column("created") - val updated = Column("updated") - val notes = Column("notes") - val folder = Column("folder_id") - val all = List( - id, - cid, - name, - itemDate, - source, - incoming, - state, - corrOrg, - corrPerson, - concPerson, - concEquipment, - inReplyTo, - dueDate, - created, - updated, - notes, - folder - ) - } - private val currentTime = Timestamp.current[ConnectionIO] From 80406cabc248e1bfb06a2208f8b6ae12ef76430c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 21:21:56 +0100 Subject: [PATCH 22/38] Refactoring some code into separate files --- .../docspell/backend/ops/OFulltext.scala | 14 +- .../scala/docspell/backend/ops/OItem.scala | 6 +- .../docspell/backend/ops/OItemSearch.scala | 26 +- .../docspell/joex/notify/MailContext.scala | 6 +- .../joex/notify/NotifyDueItemsTask.scala | 10 +- .../restserver/conv/Conversions.scala | 4 +- .../store/queries/AttachmentLight.scala | 10 + .../docspell/store/queries/CustomValue.scala | 5 + .../docspell/store/queries/ItemData.scala | 24 ++ .../store/queries/ItemFieldValue.scala | 11 + .../docspell/store/queries/ListItem.scala | 21 ++ .../store/queries/ListItemWithTags.scala | 10 + .../scala/docspell/store/queries/QItem.scala | 231 ++---------------- .../store/queries/QMoveAttachment.scala | 51 ++++ .../scala/docspell/store/queries/Query.scala | 57 +++++ .../docspell/store/queries/SelectedItem.scala | 6 + .../docspell/store/queries/TextAndTag.scala | 9 + 17 files changed, 260 insertions(+), 241 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/AttachmentLight.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/CustomValue.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/ItemData.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/ItemFieldValue.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/ListItem.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/ListItemWithTags.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/Query.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/SelectedItem.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/TextAndTag.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index f9efed22..a9fa7716 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -3,12 +3,11 @@ package docspell.backend.ops import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.ftsclient._ -import docspell.store.queries.{QFolder, QItem} +import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob import docspell.store.{Store, qb} @@ -112,15 +111,15 @@ object OFulltext { ftsItems = ftsR.results.groupBy(_.itemId) select = ftsItems.values - .map(_.sortBy(-_.score).head) - .map(r => QItem.SelectedItem(r.itemId, r.score)) + .map(_.minBy(-_.score)) + .map(r => SelectedItem(r.itemId, r.score)) .toSet itemsWithTags <- store .transact( QItem.findItemsWithTags( account.collective, - QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select) + QItem.findSelectedItems(Query.empty(account), maxNoteLen, select) ) ) .take(batch.limit.toLong) @@ -227,10 +226,9 @@ object OFulltext { ): PartialFunction[A, (A, FtsData)] = { case a if ftrItems.contains(ItemId[A].itemId(a)) => val ftsDataItems = ftrItems - .get(ItemId[A].itemId(a)) - .getOrElse(Nil) + .getOrElse(ItemId[A].itemId(a), Nil) .map(im => - FtsDataItem(im.score, im.data, ftr.highlight.get(im.id).getOrElse(Nil)) + FtsDataItem(im.score, im.data, ftr.highlight.getOrElse(im.id, Nil)) ) (a, FtsData(ftr.maxScore, ftr.count, ftr.qtime, ftsDataItems)) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 13ee91c7..889173d3 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -4,16 +4,14 @@ import cats.data.NonEmptyList import cats.data.OptionT import cats.effect.{Effect, Resource} import cats.implicits._ - import docspell.backend.JobFactory import docspell.common._ import docspell.ftsclient.FtsClient import docspell.store.UpdateResult -import docspell.store.queries.{QAttachment, QItem} +import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} import docspell.store.queue.JobQueue import docspell.store.records._ import docspell.store.{AddResult, Store} - import doobie.implicits._ import org.log4s.getLogger @@ -206,7 +204,7 @@ object OItem { target: Ident ): F[AddResult] = store - .transact(QItem.moveAttachmentBefore(itemId, source, target)) + .transact(QMoveAttachment.moveAttachmentBefore(itemId, source, target)) .attempt .map(AddResult.fromUpdate) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 94270e6f..67d9086f 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -9,7 +9,7 @@ import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.store.queries.{QAttachment, QItem} import docspell.store.records._ -import docspell.store.{Store, qb} +import docspell.store._ import bitpeace.{FileMeta, RangeDef} import doobie.implicits._ @@ -53,26 +53,26 @@ trait OItemSearch[F[_]] { object OItemSearch { - type CustomValue = QItem.CustomValue - val CustomValue = QItem.CustomValue + type CustomValue = queries.CustomValue + val CustomValue = queries.CustomValue - type Query = QItem.Query - val Query = QItem.Query + type Query = queries.Query + val Query = queries.Query type Batch = qb.Batch val Batch = docspell.store.qb.Batch - type ListItem = QItem.ListItem - val ListItem = QItem.ListItem + type ListItem = queries.ListItem + val ListItem = queries.ListItem - type ListItemWithTags = QItem.ListItemWithTags - val ListItemWithTags = QItem.ListItemWithTags + type ListItemWithTags = queries.ListItemWithTags + val ListItemWithTags = queries.ListItemWithTags - type ItemFieldValue = QItem.ItemFieldValue - val ItemFieldValue = QItem.ItemFieldValue + type ItemFieldValue = queries.ItemFieldValue + val ItemFieldValue = queries.ItemFieldValue - type ItemData = QItem.ItemData - val ItemData = QItem.ItemData + type ItemData = queries.ItemData + val ItemData = queries.ItemData trait BinaryData[F[_]] { def data: Stream[F, Byte] diff --git a/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala b/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala index 4f243521..e8097f6a 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala @@ -2,7 +2,7 @@ package docspell.joex.notify import docspell.common._ import docspell.joex.notify.YamuscaConverter._ -import docspell.store.queries.QItem +import docspell.store.queries.ListItem import yamusca.implicits._ import yamusca.imports._ @@ -19,7 +19,7 @@ case class MailContext( object MailContext { def from( - items: Vector[QItem.ListItem], + items: Vector[ListItem], max: Int, account: AccountId, itemBaseUri: Option[LenientUri], @@ -46,7 +46,7 @@ object MailContext { object ItemData { - def apply(now: Timestamp)(i: QItem.ListItem): ItemData = { + def apply(now: Timestamp)(i: ListItem): ItemData = { val dueIn = i.dueDate.map(dt => Timestamp.daysBetween(now, dt)) val dueInLabel = dueIn.map { case 0 => "**today**" diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index b4a59291..1819fac1 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -3,14 +3,12 @@ package docspell.joex.notify import cats.data.OptionT import cats.effect._ import cats.implicits._ - -import docspell.backend.ops.OItemSearch.Batch +import docspell.backend.ops.OItemSearch.{Batch, ListItem, Query} import docspell.common._ import docspell.joex.mail.EmilHeader import docspell.joex.scheduler.{Context, Task} import docspell.store.queries.QItem import docspell.store.records._ - import emil._ import emil.builder._ import emil.javamail.syntax._ @@ -66,11 +64,11 @@ object NotifyDueItemsTask { mail <- OptionT.liftF(makeMail(sendCfg, cfg, ctx.args, items)) } yield mail - def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[QItem.ListItem]] = + def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[ListItem]] = for { now <- Timestamp.current[F] q = - QItem.Query + Query .empty(ctx.args.account) .copy( states = ItemState.validStates.toList, @@ -91,7 +89,7 @@ object NotifyDueItemsTask { sendCfg: MailSendConfig, cfg: RUserEmail, args: Args, - items: Vector[QItem.ListItem] + items: Vector[ListItem] ): F[Mail[F]] = Timestamp.current[F].map { now => val templateCtx = diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index ece9ae45..f817fb10 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -16,7 +16,7 @@ import docspell.common.syntax.all._ import docspell.ftsclient.FtsResult import docspell.restapi.model._ import docspell.restserver.conv.Conversions._ -import docspell.store.queries.QItem +import docspell.store.queries.{AttachmentLight => QAttachmentLight} import docspell.store.records._ import docspell.store.{AddResult, UpdateResult} @@ -234,7 +234,7 @@ trait Conversions { customfields = i.customfields.map(mkItemFieldValue) ) - private def mkAttachmentLight(qa: QItem.AttachmentLight): AttachmentLight = + private def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight = AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount) def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = { diff --git a/modules/store/src/main/scala/docspell/store/queries/AttachmentLight.scala b/modules/store/src/main/scala/docspell/store/queries/AttachmentLight.scala new file mode 100644 index 00000000..72caee49 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/AttachmentLight.scala @@ -0,0 +1,10 @@ +package docspell.store.queries + +import docspell.common._ + +case class AttachmentLight( + id: Ident, + position: Int, + name: Option[String], + pageCount: Option[Int] +) diff --git a/modules/store/src/main/scala/docspell/store/queries/CustomValue.scala b/modules/store/src/main/scala/docspell/store/queries/CustomValue.scala new file mode 100644 index 00000000..fddaf92f --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/CustomValue.scala @@ -0,0 +1,5 @@ +package docspell.store.queries + +import docspell.common._ + +case class CustomValue(field: Ident, value: String) diff --git a/modules/store/src/main/scala/docspell/store/queries/ItemData.scala b/modules/store/src/main/scala/docspell/store/queries/ItemData.scala new file mode 100644 index 00000000..0774f9cb --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/ItemData.scala @@ -0,0 +1,24 @@ +package docspell.store.queries + +import bitpeace.FileMeta +import docspell.common._ +import docspell.store.records._ + +case class ItemData( + item: RItem, + corrOrg: Option[ROrganization], + corrPerson: Option[RPerson], + concPerson: Option[RPerson], + concEquip: Option[REquipment], + inReplyTo: Option[IdRef], + folder: Option[IdRef], + tags: Vector[RTag], + attachments: Vector[(RAttachment, FileMeta)], + sources: Vector[(RAttachmentSource, FileMeta)], + archives: Vector[(RAttachmentArchive, FileMeta)], + customFields: Vector[ItemFieldValue] +) { + + def filterCollective(coll: Ident): Option[ItemData] = + if (item.cid == coll) Some(this) else None +} diff --git a/modules/store/src/main/scala/docspell/store/queries/ItemFieldValue.scala b/modules/store/src/main/scala/docspell/store/queries/ItemFieldValue.scala new file mode 100644 index 00000000..6814f6d7 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/ItemFieldValue.scala @@ -0,0 +1,11 @@ +package docspell.store.queries + +import docspell.common._ + +case class ItemFieldValue( + fieldId: Ident, + fieldName: Ident, + fieldLabel: Option[String], + fieldType: CustomFieldType, + value: String +) diff --git a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala new file mode 100644 index 00000000..d5a37595 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala @@ -0,0 +1,21 @@ +package docspell.store.queries + +import docspell.common._ + +case class ListItem( + id: Ident, + name: String, + state: ItemState, + date: Timestamp, + dueDate: Option[Timestamp], + source: String, + direction: Direction, + created: Timestamp, + fileCount: Int, + corrOrg: Option[IdRef], + corrPerson: Option[IdRef], + concPerson: Option[IdRef], + concEquip: Option[IdRef], + folder: Option[IdRef], + notes: Option[String] +) diff --git a/modules/store/src/main/scala/docspell/store/queries/ListItemWithTags.scala b/modules/store/src/main/scala/docspell/store/queries/ListItemWithTags.scala new file mode 100644 index 00000000..5401986d --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/ListItemWithTags.scala @@ -0,0 +1,10 @@ +package docspell.store.queries + +import docspell.store.records.RTag + +case class ListItemWithTags( + item: ListItem, + tags: List[RTag], + attachments: List[AttachmentLight], + customfields: List[ItemFieldValue] +) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index feba4947..847a4bba 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -1,6 +1,5 @@ package docspell.store.queries -import cats.data.OptionT import cats.data.{NonEmptyList => Nel} import cats.effect.Sync import cats.effect.concurrent.Ref @@ -14,87 +13,28 @@ import docspell.store.qb.DSL._ import docspell.store.qb._ import docspell.store.records._ -import bitpeace.FileMeta -import doobie._ +import doobie.{Query => _, _} import doobie.implicits._ import org.log4s._ object QItem { private[this] val logger = getLogger - def moveAttachmentBefore( - itemId: Ident, - source: Ident, - target: Ident - ): ConnectionIO[Int] = { - - // rs < rt - def moveBack(rs: RAttachment, rt: RAttachment): ConnectionIO[Int] = - for { - n <- RAttachment.decPositions(itemId, rs.position, rt.position) - k <- RAttachment.updatePosition(rs.id, rt.position) - } yield n + k - - // rs > rt - def moveForward(rs: RAttachment, rt: RAttachment): ConnectionIO[Int] = - for { - n <- RAttachment.incPositions(itemId, rt.position, rs.position) - k <- RAttachment.updatePosition(rs.id, rt.position) - } yield n + k - - (for { - _ <- OptionT.liftF( - if (source == target) - Sync[ConnectionIO].raiseError(new Exception("Attachments are the same!")) - else ().pure[ConnectionIO] - ) - rs <- OptionT(RAttachment.findById(source)).filter(_.itemId == itemId) - rt <- OptionT(RAttachment.findById(target)).filter(_.itemId == itemId) - n <- OptionT.liftF( - if (rs.position == rt.position || rs.position + 1 == rt.position) - 0.pure[ConnectionIO] - else if (rs.position < rt.position) moveBack(rs, rt) - else moveForward(rs, rt) - ) - } yield n).getOrElse(0) - - } - - case class ItemFieldValue( - fieldId: Ident, - fieldName: Ident, - fieldLabel: Option[String], - fieldType: CustomFieldType, - value: String - ) - case class ItemData( - item: RItem, - corrOrg: Option[ROrganization], - corrPerson: Option[RPerson], - concPerson: Option[RPerson], - concEquip: Option[REquipment], - inReplyTo: Option[IdRef], - folder: Option[IdRef], - tags: Vector[RTag], - attachments: Vector[(RAttachment, FileMeta)], - sources: Vector[(RAttachmentSource, FileMeta)], - archives: Vector[(RAttachmentArchive, FileMeta)], - customFields: Vector[ItemFieldValue] - ) { - - def filterCollective(coll: Ident): Option[ItemData] = - if (item.cid == coll) Some(this) else None - } + private val equip = REquipment.as("e") + private val org = ROrganization.as("o") + private val pers0 = RPerson.as("pers0") + private val pers1 = RPerson.as("pers1") + private val f = RFolder.as("f") + private val i = RItem.as("i") + private val cf = RCustomField.as("cf") + private val cv = RCustomFieldValue.as("cvf") + private val a = RAttachment.as("a") + private val m = RAttachmentMeta.as("m") + private val tag = RTag.as("t") + private val ti = RTagItem.as("ti") def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { - val equip = REquipment.as("e") - val org = ROrganization.as("o") - val pers0 = RPerson.as("p0") - val pers1 = RPerson.as("p1") - val f = RFolder.as("f") - val i = RItem.as("i") - val ref = RItem.as("ref") - + val ref = RItem.as("ref") val cq = Select( select(i.all, org.all, pers0.all, pers1.all, equip.all) @@ -146,90 +86,13 @@ object QItem { def findCustomFieldValuesForItem( itemId: Ident - ): ConnectionIO[Vector[ItemFieldValue]] = { - val cf = RCustomField.as("cf") - val cv = RCustomFieldValue.as("cvf") - + ): ConnectionIO[Vector[ItemFieldValue]] = Select( select(cf.id, cf.name, cf.label, cf.ftype, cv.value), from(cv) .innerJoin(cf, cf.id === cv.field), cv.itemId === itemId ).build.query[ItemFieldValue].to[Vector] - } - - case class ListItem( - id: Ident, - name: String, - state: ItemState, - date: Timestamp, - dueDate: Option[Timestamp], - source: String, - direction: Direction, - created: Timestamp, - fileCount: Int, - corrOrg: Option[IdRef], - corrPerson: Option[IdRef], - concPerson: Option[IdRef], - concEquip: Option[IdRef], - folder: Option[IdRef], - notes: Option[String] - ) - - case class CustomValue(field: Ident, value: String) - - case class Query( - account: AccountId, - name: Option[String], - states: Seq[ItemState], - direction: Option[Direction], - corrPerson: Option[Ident], - corrOrg: Option[Ident], - concPerson: Option[Ident], - concEquip: Option[Ident], - folder: Option[Ident], - tagsInclude: List[Ident], - tagsExclude: List[Ident], - tagCategoryIncl: List[String], - tagCategoryExcl: List[String], - dateFrom: Option[Timestamp], - dateTo: Option[Timestamp], - dueDateFrom: Option[Timestamp], - dueDateTo: Option[Timestamp], - allNames: Option[String], - itemIds: Option[Set[Ident]], - customValues: Seq[CustomValue], - source: Option[String], - orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]] - ) - - object Query { - def empty(account: AccountId): Query = - Query( - account, - None, - Seq.empty, - None, - None, - None, - None, - None, - None, - Nil, - Nil, - Nil, - Nil, - None, - None, - None, - None, - None, - None, - Seq.empty, - None, - None - ) - } private def findCustomFieldValuesForColl( coll: Ident, @@ -262,13 +125,6 @@ object QItem { val num = Column[Int]("num", this) val itemId = Column[Ident]("item_id", this) } - val equip = REquipment.as("e1") - val org = ROrganization.as("o0") - val p0 = RPerson.as("p0") - val p1 = RPerson.as("p1") - val f = RFolder.as("f1") - val i = RItem.as("i") - val a = RAttachment.as("a") val coll = q.account.collective @@ -285,10 +141,10 @@ object QItem { coalesce(Attachs.num.s, lit(0)).s, org.oid.s, org.name.s, - p0.pid.s, - p0.name.s, - p1.pid.s, - p1.name.s, + pers0.pid.s, + pers0.name.s, + pers1.pid.s, + pers1.name.s, equip.eid.s, equip.name.s, f.id.s, @@ -311,9 +167,9 @@ object QItem { Attachs.aliasName, //alias, todo improve dsl Attachs.itemId === i.id ) - .leftJoin(p0, p0.pid === i.corrPerson && p0.cid === coll) + .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll) .leftJoin(org, org.oid === i.corrOrg && org.cid === coll) - .leftJoin(p1, p1.pid === i.concPerson && p1.cid === coll) + .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll) .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll), where( i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) && @@ -338,13 +194,6 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - val equip = REquipment.as("e1") - val org = ROrganization.as("o0") - val pers0 = RPerson.as("p0") - val pers1 = RPerson.as("p1") - val f = RFolder.as("f1") - val i = RItem.as("i") - val cond: Condition => Condition = c => c &&? @@ -386,7 +235,6 @@ object QItem { sql.query[ListItem].stream } - case class SelectedItem(itemId: Ident, weight: Double) def findSelectedItems( q: Query, maxNoteLen: Int, @@ -427,19 +275,6 @@ object QItem { from.query[ListItem].stream } - case class AttachmentLight( - id: Ident, - position: Int, - name: Option[String], - pageCount: Option[Int] - ) - case class ListItemWithTags( - item: ListItem, - tags: List[RTag], - attachments: List[AttachmentLight], - customfields: List[ItemFieldValue] - ) - /** Same as `findItems` but resolves the tags for each item. Note that * this is implemented by running an additional query per item. */ @@ -482,17 +317,13 @@ object QItem { ) } - private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = { - val a = RAttachment.as("a") - val m = RAttachmentMeta.as("m") - + private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = Select( select(a.id, a.position, a.name, m.pages), from(a) .leftJoin(m, m.id === a.id), a.itemId === item ).build.query[AttachmentLight].to[List] - } def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = for { @@ -602,21 +433,12 @@ object QItem { .streamWithChunkSize(chunkSize) } - case class TagName(id: Ident, name: String) - case class TextAndTag(itemId: Ident, text: String, tag: Option[TagName]) - def resolveTextAndTag( collective: Ident, itemId: Ident, tagCategory: String, pageSep: String ): ConnectionIO[TextAndTag] = { - val tag = RTag.as("t") - val a = RAttachment.as("a") - val am = RAttachmentMeta.as("m") - val ti = RTagItem.as("ti") - val i = RItem.as("i") - val tags = TableDef("tags").as("tt") val tagsItem = Column[Ident]("itemid", tags) val tagsTid = Column[Ident]("tid", tags) @@ -632,12 +454,12 @@ object QItem { ) )( Select( - select(am.content, tagsTid, tagsName), + select(m.content, tagsTid, tagsName), from(i) .innerJoin(a, a.itemId === i.id) - .innerJoin(am, a.id === am.id) + .innerJoin(m, a.id === m.id) .leftJoin(tags, tagsItem === i.id), - i.id === itemId && i.cid === collective && am.content.isNotNull && am.content <> "" + i.id === itemId && i.cid === collective && m.content.isNotNull && m.content <> "" ) ).build @@ -645,7 +467,7 @@ object QItem { _ <- logger.ftrace[ConnectionIO]( s"query: $q (${itemId.id}, ${collective.id}, ${tagCategory})" ) - texts <- q.query[(String, Option[TagName])].to[List] + texts <- q.query[(String, Option[TextAndTag.TagName])].to[List] _ <- logger.ftrace[ConnectionIO]( s"Got ${texts.size} text and tag entries for item ${itemId.id}" ) @@ -653,5 +475,4 @@ object QItem { txt = texts.map(_._1).mkString(pageSep) } yield TextAndTag(itemId, txt, tag) } - } diff --git a/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala new file mode 100644 index 00000000..c3463cb5 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala @@ -0,0 +1,51 @@ +package docspell.store.queries + +import cats.effect._ +import cats.data.OptionT +import cats.implicits._ + +import docspell.common._ +import docspell.store.records._ + +import doobie.{Query => _, _} +import doobie.implicits._ + +object QMoveAttachment { + def moveAttachmentBefore( + itemId: Ident, + source: Ident, + target: Ident + ): ConnectionIO[Int] = { + + // rs < rt + def moveBack(rs: RAttachment, rt: RAttachment): ConnectionIO[Int] = + for { + n <- RAttachment.decPositions(itemId, rs.position, rt.position) + k <- RAttachment.updatePosition(rs.id, rt.position) + } yield n + k + + // rs > rt + def moveForward(rs: RAttachment, rt: RAttachment): ConnectionIO[Int] = + for { + n <- RAttachment.incPositions(itemId, rt.position, rs.position) + k <- RAttachment.updatePosition(rs.id, rt.position) + } yield n + k + + (for { + _ <- OptionT.liftF( + if (source == target) + Sync[ConnectionIO].raiseError(new Exception("Attachments are the same!")) + else ().pure[ConnectionIO] + ) + rs <- OptionT(RAttachment.findById(source)).filter(_.itemId == itemId) + rt <- OptionT(RAttachment.findById(target)).filter(_.itemId == itemId) + n <- OptionT.liftF( + if (rs.position == rt.position || rs.position + 1 == rt.position) + 0.pure[ConnectionIO] + else if (rs.position < rt.position) moveBack(rs, rt) + else moveForward(rs, rt) + ) + } yield n).getOrElse(0) + + } +} diff --git a/modules/store/src/main/scala/docspell/store/queries/Query.scala b/modules/store/src/main/scala/docspell/store/queries/Query.scala new file mode 100644 index 00000000..0d68bdef --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/Query.scala @@ -0,0 +1,57 @@ +package docspell.store.queries + +import docspell.common._ +import docspell.store.records.RItem + +case class Query( + account: AccountId, + name: Option[String], + states: Seq[ItemState], + direction: Option[Direction], + corrPerson: Option[Ident], + corrOrg: Option[Ident], + concPerson: Option[Ident], + concEquip: Option[Ident], + folder: Option[Ident], + tagsInclude: List[Ident], + tagsExclude: List[Ident], + tagCategoryIncl: List[String], + tagCategoryExcl: List[String], + dateFrom: Option[Timestamp], + dateTo: Option[Timestamp], + dueDateFrom: Option[Timestamp], + dueDateTo: Option[Timestamp], + allNames: Option[String], + itemIds: Option[Set[Ident]], + customValues: Seq[CustomValue], + source: Option[String], + orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]] +) + +object Query { + def empty(account: AccountId): Query = + Query( + account, + None, + Seq.empty, + None, + None, + None, + None, + None, + None, + Nil, + Nil, + Nil, + Nil, + None, + None, + None, + None, + None, + None, + Seq.empty, + None, + None + ) +} diff --git a/modules/store/src/main/scala/docspell/store/queries/SelectedItem.scala b/modules/store/src/main/scala/docspell/store/queries/SelectedItem.scala new file mode 100644 index 00000000..9beba7cb --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/SelectedItem.scala @@ -0,0 +1,6 @@ +package docspell.store.queries + +import docspell.common._ + +/** Some preselected item from a fulltext search. */ +case class SelectedItem(itemId: Ident, weight: Double) diff --git a/modules/store/src/main/scala/docspell/store/queries/TextAndTag.scala b/modules/store/src/main/scala/docspell/store/queries/TextAndTag.scala new file mode 100644 index 00000000..ffeb2984 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/TextAndTag.scala @@ -0,0 +1,9 @@ +package docspell.store.queries + +import docspell.common._ + +case class TextAndTag(itemId: Ident, text: String, tag: Option[TextAndTag.TagName]) + +object TextAndTag { + case class TagName(id: Ident, name: String) +} From 2dff686fa0b16ac0ffb3f6907837132dc90a55a6 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Mon, 14 Dec 2020 23:07:06 +0100 Subject: [PATCH 23/38] Introduce unit condition --- .../docspell/backend/ops/OFulltext.scala | 1 + .../scala/docspell/backend/ops/OItem.scala | 2 + .../docspell/backend/ops/OItemSearch.scala | 2 +- .../joex/notify/NotifyDueItemsTask.scala | 2 + .../scala/docspell/store/qb/Condition.scala | 45 +++++++-- .../main/scala/docspell/store/qb/DSL.scala | 8 +- .../main/scala/docspell/store/qb/Select.scala | 19 ++-- .../store/qb/impl/ConditionBuilder.scala | 68 +++++++++++-- .../store/qb/impl/SelectBuilder.scala | 9 +- .../docspell/store/queries/ItemData.scala | 3 +- .../scala/docspell/store/queries/QItem.scala | 2 +- .../store/queries/QMoveAttachment.scala | 4 +- .../docspell/store/qb/QueryBuilderTest.scala | 2 +- .../store/qb/impl/ConditionBuilderTest.scala | 97 +++++++++++++++++++ 14 files changed, 225 insertions(+), 39 deletions(-) create mode 100644 modules/store/src/test/scala/docspell/store/qb/impl/ConditionBuilderTest.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index a9fa7716..d7f416d2 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -3,6 +3,7 @@ package docspell.backend.ops import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 889173d3..bcefe0e5 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -4,6 +4,7 @@ import cats.data.NonEmptyList import cats.data.OptionT import cats.effect.{Effect, Resource} import cats.implicits._ + import docspell.backend.JobFactory import docspell.common._ import docspell.ftsclient.FtsClient @@ -12,6 +13,7 @@ import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} import docspell.store.queue.JobQueue import docspell.store.records._ import docspell.store.{AddResult, Store} + import doobie.implicits._ import org.log4s.getLogger diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 67d9086f..ccfe6230 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -7,9 +7,9 @@ import fs2.Stream import docspell.backend.ops.OItemSearch._ import docspell.common._ +import docspell.store._ import docspell.store.queries.{QAttachment, QItem} import docspell.store.records._ -import docspell.store._ import bitpeace.{FileMeta, RangeDef} import doobie.implicits._ diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index 1819fac1..78ec0882 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -3,12 +3,14 @@ package docspell.joex.notify import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.backend.ops.OItemSearch.{Batch, ListItem, Query} import docspell.common._ import docspell.joex.mail.EmilHeader import docspell.joex.scheduler.{Context, Task} import docspell.store.queries.QItem import docspell.store.records._ + import emil._ import emil.builder._ import emil.javamail.syntax._ diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 2a1f3097..0b0c0692 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -7,6 +7,9 @@ import doobie._ sealed trait Condition object Condition { + case object UnitCondition extends Condition + + val unit: Condition = UnitCondition case class CompareVal[A](column: Column[A], op: Operator, value: A)(implicit val P: Put[A] @@ -26,36 +29,58 @@ object Condition { case class IsNull(col: Column[_]) extends Condition - case class And(c: Condition, cs: Vector[Condition]) extends Condition { + case class And(inner: NonEmptyList[Condition]) extends Condition { def append(other: Condition): And = other match { - case And(oc, ocs) => - And(c, cs ++ (oc +: ocs)) + case And(otherInner) => + And(inner.concatNel(otherInner)) case _ => - And(c, cs :+ other) + And(inner.append(other)) } } object And { def apply(c: Condition, cs: Condition*): And = - And(c, cs.toVector) + And(NonEmptyList(c, cs.toList)) + object Inner extends InnerCondition { + def unapply(node: Condition): Option[NonEmptyList[Condition]] = + node match { + case n: And => + Option(n.inner) + case _ => + None + } + } } - case class Or(c: Condition, cs: Vector[Condition]) extends Condition { + case class Or(inner: NonEmptyList[Condition]) extends Condition { def append(other: Condition): Or = other match { - case Or(oc, ocs) => - Or(c, cs ++ (oc +: ocs)) + case Or(otherInner) => + Or(inner.concatNel(otherInner)) case _ => - Or(c, cs :+ other) + Or(inner.append(other)) } } object Or { def apply(c: Condition, cs: Condition*): Or = - Or(c, cs.toVector) + Or(NonEmptyList(c, cs.toList)) + + object Inner extends InnerCondition { + def unapply(node: Condition): Option[NonEmptyList[Condition]] = + node match { + case n: Or => + Option(n.inner) + case _ => + None + } + } } case class Not(c: Condition) extends Condition object Not {} + trait InnerCondition { + def unapply(node: Condition): Option[NonEmptyList[Condition]] + } } diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index 97c80a2a..fe3bda42 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -97,15 +97,15 @@ trait DSL extends DoobieMeta { case a: Condition.And => cs.foldLeft(a)(_.append(_)) case _ => - Condition.And(c, cs.toVector) + Condition.And(c, cs: _*) } def or(c: Condition, cs: Condition*): Condition = c match { - case Condition.Or(head, tail) => - Condition.Or(head, tail ++ (c +: cs.toVector)) + case o: Condition.Or => + cs.foldLeft(o)(_.append(_)) case _ => - Condition.Or(c, cs.toVector) + Condition.Or(c, cs: _*) } def not(c: Condition): Condition = diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index 155109a8..ca64d218 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -46,35 +46,35 @@ sealed trait Select { object Select { def apply(projection: Nel[SelectExpr], from: FromExpr) = - SimpleSelect(false, projection, from, None, None) + SimpleSelect(false, projection, from, Condition.unit, None) def apply(projection: SelectExpr, from: FromExpr) = - SimpleSelect(false, Nel.of(projection), from, None, None) + SimpleSelect(false, Nel.of(projection), from, Condition.unit, None) def apply( projection: Nel[SelectExpr], from: FromExpr, where: Condition - ) = SimpleSelect(false, projection, from, Some(where), None) + ) = SimpleSelect(false, projection, from, where, None) def apply( projection: SelectExpr, from: FromExpr, where: Condition - ) = SimpleSelect(false, Nel.of(projection), from, Some(where), None) + ) = SimpleSelect(false, Nel.of(projection), from, where, None) def apply( projection: Nel[SelectExpr], from: FromExpr, where: Condition, groupBy: GroupBy - ) = SimpleSelect(false, projection, from, Some(where), Some(groupBy)) + ) = SimpleSelect(false, projection, from, where, Some(groupBy)) case class SimpleSelect( distinctFlag: Boolean, projection: Nel[SelectExpr], from: FromExpr, - where: Option[Condition], + where: Condition, groupBy: Option[GroupBy] ) extends Select { def group(gb: GroupBy): SimpleSelect = @@ -84,9 +84,10 @@ object Select { copy(distinctFlag = true) def where(c: Option[Condition]): SimpleSelect = - copy(where = c) + where(c.getOrElse(Condition.unit)) + def where(c: Condition): SimpleSelect = - copy(where = Some(c)) + copy(where = c) def appendSelect(e: SelectExpr): SimpleSelect = copy(projection = projection.append(e)) @@ -95,7 +96,7 @@ object Select { copy(from = f(from)) def changeWhere(f: Condition => Condition): SimpleSelect = - copy(where = where.map(f)) + copy(where = f(where)) def orderBy(ob: OrderBy, obs: OrderBy*): Select = Ordered(this, ob, obs.toVector) diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index cc700b18..3a4e4ede 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -1,5 +1,7 @@ package docspell.store.qb.impl +import cats.data.NonEmptyList + import docspell.store.qb._ import _root_.doobie.implicits._ @@ -12,8 +14,55 @@ object ConditionBuilder { val parenOpen = Fragment.const0("(") val parenClose = Fragment.const0(")") - def build(expr: Condition): Fragment = - expr match { + final def reduce(c: Condition): Condition = + c match { + case Condition.And(inner) => + NonEmptyList.fromList(flatten(inner.toList, Condition.And.Inner)) match { + case Some(rinner) => + if (rinner.tail.isEmpty) reduce(rinner.head) + else Condition.And(rinner.reverse.map(reduce)) + case None => + Condition.unit + } + + case Condition.Or(inner) => + NonEmptyList.fromList(flatten(inner.toList, Condition.Or.Inner)) match { + case Some(rinner) => + if (rinner.tail.isEmpty) reduce(rinner.head) + else Condition.Or(rinner.reverse.map(reduce)) + case None => + Condition.unit + } + + case Condition.Not(Condition.UnitCondition) => + Condition.unit + + case Condition.Not(Condition.Not(inner)) => + reduce(inner) + + case _ => + c + } + + private def flatten( + els: List[Condition], + nodePattern: Condition.InnerCondition, + result: List[Condition] = Nil + ): List[Condition] = + els match { + case Nil => + result + case nodePattern(more) :: tail => + val spliced = flatten(more.toList, nodePattern, result) + flatten(tail, nodePattern, spliced) + case Condition.UnitCondition :: tail => + flatten(tail, nodePattern, result) + case h :: tail => + flatten(tail, nodePattern, h :: result) + } + + final def build(expr: Condition): Fragment = + reduce(expr) match { case c @ Condition.CompareVal(col, op, value) => val opFrag = operator(op) val valFrag = buildValue(value)(c.P) @@ -58,14 +107,14 @@ object ConditionBuilder { case Condition.IsNull(col) => SelectExprBuilder.column(col) ++ fr" is null" - case Condition.And(c, cs) => - val inner = cs.prepended(c).map(build).reduce(_ ++ and ++ _) - if (cs.isEmpty) inner + case Condition.And(ands) => + val inner = ands.map(build).reduceLeft(_ ++ and ++ _) + if (ands.tail.isEmpty) inner else parenOpen ++ inner ++ parenClose - case Condition.Or(c, cs) => - val inner = cs.prepended(c).map(build).reduce(_ ++ or ++ _) - if (cs.isEmpty) inner + case Condition.Or(ors) => + val inner = ors.map(build).reduceLeft(_ ++ or ++ _) + if (ors.tail.isEmpty) inner else parenOpen ++ inner ++ parenClose case Condition.Not(Condition.IsNull(col)) => @@ -73,6 +122,9 @@ object ConditionBuilder { case Condition.Not(c) => fr"NOT" ++ build(c) + + case Condition.UnitCondition => + Fragment.empty } def operator(op: Operator): Fragment = diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala index 68743737..c6b92a7d 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala @@ -47,7 +47,7 @@ object SelectBuilder { def buildSimple(sq: Select.SimpleSelect): Fragment = { val f0 = sq.projection.map(selectExpr).reduceLeft(_ ++ comma ++ _) val f1 = fromExpr(sq.from) - val f2 = sq.where.map(cond).getOrElse(Fragment.empty) + val f2 = cond(sq.where) val f3 = sq.groupBy.map(groupBy).getOrElse(Fragment.empty) f0 ++ f1 ++ f2 ++ f3 } @@ -70,7 +70,12 @@ object SelectBuilder { FromExprBuilder.build(fr) def cond(c: Condition): Fragment = - fr" WHERE" ++ ConditionBuilder.build(c) + c match { + case Condition.UnitCondition => + Fragment.empty + case _ => + fr" WHERE" ++ ConditionBuilder.build(c) + } def groupBy(gb: GroupBy): Fragment = { val f0 = gb.names.prepended(gb.name).map(selectExpr).reduce(_ ++ comma ++ _) diff --git a/modules/store/src/main/scala/docspell/store/queries/ItemData.scala b/modules/store/src/main/scala/docspell/store/queries/ItemData.scala index 0774f9cb..d96d726e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/ItemData.scala +++ b/modules/store/src/main/scala/docspell/store/queries/ItemData.scala @@ -1,9 +1,10 @@ package docspell.store.queries -import bitpeace.FileMeta import docspell.common._ import docspell.store.records._ +import bitpeace.FileMeta + case class ItemData( item: RItem, corrOrg: Option[ROrganization], diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 847a4bba..1d82714d 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -13,8 +13,8 @@ import docspell.store.qb.DSL._ import docspell.store.qb._ import docspell.store.records._ -import doobie.{Query => _, _} import doobie.implicits._ +import doobie.{Query => _, _} import org.log4s._ object QItem { diff --git a/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala index c3463cb5..8992065e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QMoveAttachment.scala @@ -1,14 +1,14 @@ package docspell.store.queries -import cats.effect._ import cats.data.OptionT +import cats.effect._ import cats.implicits._ import docspell.common._ import docspell.store.records._ -import doobie.{Query => _, _} import doobie.implicits._ +import doobie.{Query => _, _} object QMoveAttachment { def moveAttachmentBefore( diff --git a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala index 7d5dc64d..edbe8769 100644 --- a/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala +++ b/modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala @@ -65,7 +65,7 @@ object QueryBuilderTest extends SimpleTestSuite { fail("Unexpected result") } assertEquals(group, None) - assert(where.isDefined) + assert(where != Condition.unit) case _ => fail("Unexpected case") } diff --git a/modules/store/src/test/scala/docspell/store/qb/impl/ConditionBuilderTest.scala b/modules/store/src/test/scala/docspell/store/qb/impl/ConditionBuilderTest.scala new file mode 100644 index 00000000..1e035f71 --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/qb/impl/ConditionBuilderTest.scala @@ -0,0 +1,97 @@ +package docspell.store.qb.impl + +import minitest._ +import docspell.store.qb._ +import docspell.store.qb.DSL._ +import docspell.store.qb.model.{CourseRecord, PersonRecord} + +object ConditionBuilderTest extends SimpleTestSuite { + + val c = CourseRecord.as("c") + val p = PersonRecord.as("p") + + test("reduce ands") { + val cond = + c.lessons > 3 && (c.id === 5L && (p.name === "john" && Condition.unit && p.id === 1L)) + val expected = + and(c.lessons > 3, c.id === 5L, p.name === "john", p.id === 1L) + + assertEquals(ConditionBuilder.reduce(cond), expected) + assertEquals(ConditionBuilder.reduce(expected), expected) + } + + test("reduce ors") { + val cond = + c.lessons > 3 || (c.id === 5L || (p.name === "john" || Condition.unit || p.id === 1L)) + val expected = + or(c.lessons > 3, c.id === 5L, p.name === "john", p.id === 1L) + + assertEquals(ConditionBuilder.reduce(cond), expected) + assertEquals(ConditionBuilder.reduce(expected), expected) + } + + test("mixed and / or") { + val cond = c.lessons > 3 && (p.name === "john" || p.name === "mara") && c.id > 3 + val expected = + and(c.lessons > 3, or(p.name === "john", p.name === "mara"), c.id > 3) + assertEquals(ConditionBuilder.reduce(cond), expected) + assertEquals(ConditionBuilder.reduce(expected), expected) + } + + test("reduce double not") { + val cond = Condition.Not(Condition.Not(c.name === "scala")) + assertEquals(ConditionBuilder.reduce(cond), c.name === "scala") + } + + test("reduce triple not") { + val cond = Condition.Not(Condition.Not(Condition.Not(c.name === "scala"))) + assertEquals(ConditionBuilder.reduce(cond), not(c.name === "scala")) + } + + test("reduce not to unit") { + val cond = Condition.Not(Condition.Not(Condition.Not(Condition.Not(Condition.unit)))) + assertEquals(ConditionBuilder.reduce(cond), Condition.unit) + } + + test("remove units in and/or") { + val cond = + c.name === "scala" && Condition.unit && (c.name === "fp" || Condition.unit) && Condition.unit + assertEquals(ConditionBuilder.reduce(cond), and(c.name === "scala", c.name === "fp")) + } + + test("unwrap single and/ors") { + assertEquals( + ConditionBuilder.reduce(Condition.Or(c.name === "scala")), + c.name === "scala" + ) + assertEquals( + ConditionBuilder.reduce(Condition.And(c.name === "scala")), + c.name === "scala" + ) + + assertEquals( + ConditionBuilder.reduce(Condition.unit && c.name === "scala" && Condition.unit), + c.name === "scala" + ) + assertEquals( + ConditionBuilder.reduce(Condition.unit || c.name === "scala" || Condition.unit), + c.name === "scala" + ) + + assertEquals( + ConditionBuilder.reduce(and(and(and(c.name === "scala"), Condition.unit))), + c.name === "scala" + ) + } + + test("reduce empty and/or") { + assertEquals( + ConditionBuilder.reduce(Condition.unit && Condition.unit && Condition.unit), + Condition.unit + ) + assertEquals( + ConditionBuilder.reduce(Condition.unit || Condition.unit || Condition.unit), + Condition.unit + ) + } +} From f1c4b4adb04d990542d7d2462522ef2ed0bd9f8f Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 20:16:19 +0100 Subject: [PATCH 24/38] Extract find-item query condition --- .../main/scala/docspell/store/qb/Select.scala | 10 +-- .../scala/docspell/store/queries/QItem.scala | 69 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index ca64d218..227c6266 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -170,17 +170,17 @@ object Select { } - case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select { + case class WithCte(cte: CteBind, ctes: Vector[CteBind], q: Select) extends Select { def appendSelect(e: SelectExpr): WithCte = - copy(query = query.appendSelect(e)) + copy(q = q.appendSelect(e)) def changeFrom(f: FromExpr => FromExpr): WithCte = - copy(query = query.changeFrom(f)) + copy(q = q.changeFrom(f)) def changeWhere(f: Condition => Condition): WithCte = - copy(query = query.changeWhere(f)) + copy(q = q.changeWhere(f)) def orderBy(ob: OrderBy, obs: OrderBy*): WithCte = - copy(query = query.orderBy(ob, obs: _*)) + copy(q = q.orderBy(ob, obs: _*)) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 1d82714d..456fc69f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -164,7 +164,7 @@ object QItem { i.cid === q.account.collective, GroupBy(a.itemId) ), - Attachs.aliasName, //alias, todo improve dsl + Attachs.aliasName, Attachs.itemId === i.id ) .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll) @@ -189,46 +189,45 @@ object QItem { } } + def queryCondition(q: Query): Condition = + Condition.unit &&? + q.direction.map(d => i.incoming === d) &&? + q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? + q.allNames + .map(QueryWildcard.lower) + .map(n => + org.name.like(n) || + pers0.name.like(n) || + pers1.name.like(n) || + equip.name.like(n) || + i.name.like(n) || + i.notes.like(n) + ) &&? + q.corrPerson.map(p => pers0.pid === p) &&? + q.corrOrg.map(o => org.oid === o) &&? + q.concPerson.map(p => pers1.pid === p) &&? + q.concEquip.map(e => equip.eid === e) &&? + q.folder.map(fid => f.id === fid) &&? + q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&? + q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&? + q.dueDateFrom.map(d => i.dueDate > d) &&? + q.dueDateTo.map(d => i.dueDate < d) &&? + q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? + q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&? + TagItemName + .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) + .map(subsel => i.id.in(subsel)) &&? + TagItemName + .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) + .map(subsel => i.id.notIn(subsel)) + def findItems( q: Query, maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - val cond: Condition => Condition = - c => - c &&? - q.direction.map(d => i.incoming === d) &&? - q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? - q.allNames - .map(QueryWildcard.lower) - .map(n => - org.name.like(n) || - pers0.name.like(n) || - pers1.name.like(n) || - equip.name.like(n) || - i.name.like(n) || - i.notes.like(n) - ) &&? - q.corrPerson.map(p => pers0.pid === p) &&? - q.corrOrg.map(o => org.oid === o) &&? - q.concPerson.map(p => pers1.pid === p) &&? - q.concEquip.map(e => equip.eid === e) &&? - q.folder.map(fid => f.id === fid) &&? - q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&? - q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&? - q.dueDateFrom.map(d => i.dueDate > d) &&? - q.dueDateTo.map(d => i.dueDate < d) &&? - q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? - q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&? - TagItemName - .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) - .map(subsel => i.id.in(subsel)) &&? - TagItemName - .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) - .map(subsel => i.id.notIn(subsel)) - val sql = findItemsBase(q, maxNoteLen) - .changeWhere(cond) + .changeWhere(c => c && queryCondition(q)) .limit(batch) .build logger.trace(s"List $batch items: $sql") From 56d6d2e2acc1b4223dc91a7d07b52076ad550357 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 22:12:44 +0100 Subject: [PATCH 25/38] Allow changing more parts of a select --- .../scala/docspell/store/qb/FromExpr.scala | 20 +++++ .../main/scala/docspell/store/qb/Select.scala | 84 +++++++++++++++++-- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala index 72486323..2edb13a4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala +++ b/modules/store/src/main/scala/docspell/store/qb/FromExpr.scala @@ -15,6 +15,16 @@ sealed trait FromExpr { def leftJoin(sel: Select, alias: String, on: Condition): Joined = leftJoin(Relation.SubSelect(sel, alias), on) + + /** Prepends the given from expression to existing joins. It will + * replace the current [[FromExpr.From]] value. + * + * If this is a [[FromExpr.From]], it is replaced by the given + * expression. If this is a [[FromExpr.Joined]] then the given + * expression replaces the current `From` and the joins are + * prepended to the existing joins. + */ + def prepend(fe: FromExpr): FromExpr } object FromExpr { @@ -25,6 +35,9 @@ object FromExpr { def leftJoin(other: Relation, on: Condition): Joined = Joined(this, Vector(Join.LeftJoin(other, on))) + + def prepend(fe: FromExpr): FromExpr = + fe } object From { @@ -42,6 +55,13 @@ object FromExpr { def leftJoin(other: Relation, on: Condition): Joined = Joined(from, joins :+ Join.LeftJoin(other, on)) + def prepend(fe: FromExpr): FromExpr = + fe match { + case f: From => + Joined(f, joins) + case Joined(f, js) => + Joined(f, js ++ joins) + } } sealed trait Relation diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index 227c6266..d6c17113 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -6,18 +6,32 @@ import docspell.store.qb.impl.SelectBuilder import doobie._ +/** A sql select statement that allows to change certain parts of the query. + */ sealed trait Select { + + /** Builds the sql select statement into a doobie fragment. + */ def build: Fragment = SelectBuilder(this) + /** When using this as a sub-select, an alias is required. + */ def as(alias: String): SelectExpr.SelectQuery = SelectExpr.SelectQuery(this, Some(alias)) + /** Adds one or more order-by definitions */ def orderBy(ob: OrderBy, obs: OrderBy*): Select + /** Uses the given column for ordering asc */ def orderBy(c: Column[_]): Select = orderBy(OrderBy(SelectExpr.SelectColumn(c, None), OrderBy.OrderType.Asc)) + def groupBy(gb: GroupBy): Select + + def groupBy(c: Column[_]): Select = + groupBy(GroupBy(c)) + def limit(batch: Batch): Select = this match { case Select.Limit(q, _) => @@ -39,9 +53,16 @@ sealed trait Select { def appendSelect(e: SelectExpr): Select + def withSelect(e: Nel[SelectExpr]): Select + def changeFrom(f: FromExpr => FromExpr): Select def changeWhere(f: Condition => Condition): Select + + def where(c: Option[Condition]): Select = + where(c.getOrElse(Condition.unit)) + + def where(c: Condition): Select } object Select { @@ -77,21 +98,21 @@ object Select { where: Condition, groupBy: Option[GroupBy] ) extends Select { - def group(gb: GroupBy): SimpleSelect = + def groupBy(gb: GroupBy): SimpleSelect = copy(groupBy = Some(gb)) def distinct: SimpleSelect = copy(distinctFlag = true) - def where(c: Option[Condition]): SimpleSelect = - where(c.getOrElse(Condition.unit)) - def where(c: Condition): SimpleSelect = copy(where = c) def appendSelect(e: SelectExpr): SimpleSelect = copy(projection = projection.append(e)) + def withSelect(es: Nel[SelectExpr]): SimpleSelect = + copy(projection = es) + def changeFrom(f: FromExpr => FromExpr): SimpleSelect = copy(from = f(from)) @@ -103,8 +124,11 @@ object Select { } case class RawSelect(fragment: Fragment) extends Select { + def groupBy(gb: GroupBy): Select = + sys.error("RawSelect doesn't support adding group by clause") + def appendSelect(e: SelectExpr): RawSelect = - sys.error("RawSelect doesn't support appending select expressions") + sys.error("RawSelect doesn't support appending to select list") def changeFrom(f: FromExpr => FromExpr): Select = sys.error("RawSelect doesn't support changing from expression") @@ -114,9 +138,18 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = sys.error("RawSelect doesn't support adding orderBy clause") + + def where(c: Condition): Select = + sys.error("RawSelect doesn't support adding where clause") + + def withSelect(es: Nel[SelectExpr]): Select = + sys.error("RawSelect doesn't support changing select list") } case class Union(q: Select, qs: Vector[Select]) extends Select { + def groupBy(gb: GroupBy): Union = + copy(q = q.groupBy(gb)) + def appendSelect(e: SelectExpr): Union = copy(q = q.appendSelect(e)) @@ -128,9 +161,18 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = Ordered(this, ob, obs.toVector) + + def where(c: Condition): Union = + copy(q = q.where(c)) + + def withSelect(es: Nel[SelectExpr]): Union = + copy(q = q.withSelect(es)) } case class Intersect(q: Select, qs: Vector[Select]) extends Select { + def groupBy(gb: GroupBy): Intersect = + copy(q = q.groupBy(gb)) + def appendSelect(e: SelectExpr): Intersect = copy(q = q.appendSelect(e)) @@ -142,10 +184,19 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = Ordered(this, ob, obs.toVector) + + def where(c: Condition): Intersect = + copy(q = q.where(c)) + + def withSelect(es: Nel[SelectExpr]): Intersect = + copy(q = q.withSelect(es)) } case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) extends Select { + def groupBy(gb: GroupBy): Ordered = + copy(q = q.groupBy(gb)) + def appendSelect(e: SelectExpr): Ordered = copy(q = q.appendSelect(e)) def changeFrom(f: FromExpr => FromExpr): Ordered = @@ -155,9 +206,18 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = Ordered(q, ob, obs.toVector) + + def where(c: Condition): Ordered = + copy(q = q.where(c)) + + def withSelect(es: Nel[SelectExpr]): Ordered = + copy(q = q.withSelect(es)) } case class Limit(q: Select, batch: Batch) extends Select { + def groupBy(gb: GroupBy): Limit = + copy(q = q.groupBy(gb)) + def appendSelect(e: SelectExpr): Limit = copy(q = q.appendSelect(e)) def changeFrom(f: FromExpr => FromExpr): Limit = @@ -168,9 +228,17 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): Limit = copy(q = q.orderBy(ob, obs: _*)) + def where(c: Condition): Limit = + copy(q = q.where(c)) + + def withSelect(es: Nel[SelectExpr]): Limit = + copy(q = q.withSelect(es)) } case class WithCte(cte: CteBind, ctes: Vector[CteBind], q: Select) extends Select { + def groupBy(gb: GroupBy): WithCte = + copy(q = q.groupBy(gb)) + def appendSelect(e: SelectExpr): WithCte = copy(q = q.appendSelect(e)) @@ -182,5 +250,11 @@ object Select { def orderBy(ob: OrderBy, obs: OrderBy*): WithCte = copy(q = q.orderBy(ob, obs: _*)) + + def where(c: Condition): WithCte = + copy(q = q.where(c)) + + def withSelect(es: Nel[SelectExpr]): WithCte = + copy(q = q.withSelect(es)) } } From 4ca6dfccae8a788d6ebeef7194ffaeb1eb6cdc16 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 22:14:03 +0100 Subject: [PATCH 26/38] Get basic search summary --- .../docspell/backend/ops/OCollective.scala | 4 ++-- .../docspell/backend/ops/OItemSearch.scala | 11 +++++++++ .../docspell/store/queries/QCollective.scala | 2 -- .../scala/docspell/store/queries/QItem.scala | 24 +++++++++++++++++++ .../store/queries/SearchSummary.scala | 3 +++ .../docspell/store/queries/TagCount.scala | 5 ++++ 6 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/TagCount.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index a4f06986..f68be8b7 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -66,8 +66,8 @@ trait OCollective[F[_]] { object OCollective { - type TagCount = QCollective.TagCount - val TagCount = QCollective.TagCount + type TagCount = docspell.store.queries.TagCount + val TagCount = docspell.store.queries.TagCount type InsightData = QCollective.InsightData val insightData = QCollective.InsightData diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index ccfe6230..fbfa69d2 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -24,6 +24,8 @@ trait OItemSearch[F[_]] { maxNoteLen: Int )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] + def findItemsSummary(q: Query): F[SearchSummary] + def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] def findAttachmentSource( @@ -53,6 +55,9 @@ trait OItemSearch[F[_]] { object OItemSearch { + type SearchSummary = queries.SearchSummary + val SearchSummary = queries.SearchSummary + type CustomValue = queries.CustomValue val CustomValue = queries.CustomValue @@ -139,6 +144,12 @@ object OItemSearch { .toVector } + def findItemsSummary(q: Query): F[SearchSummary] = + for { + tags <- store.transact(QItem.searchTagSummary(q)) + count <- store.transact(QItem.searchCountSummary(q)) + } yield SearchSummary(count, tags) + def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store .transact(RAttachment.findByIdAndCollective(id, collective)) diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index b9e8f74a..696d2294 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -33,8 +33,6 @@ object QCollective { } yield Names(orgs.map(_.name), pers.map(_.name), equp.map(_.name))) .getOrElse(Names.empty) - case class TagCount(tag: RTag, count: Int) - case class InsightData( incoming: Int, outgoing: Int, diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 456fc69f..fab67704 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -234,6 +234,30 @@ object QItem { sql.query[ListItem].stream } + def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { + val tagFrom = + from(ti) + .innerJoin(tag, tag.tid === ti.tagId) + .innerJoin(i, i.id === ti.itemId) + + findItemsBase(q, 0) + .withSelect(select(tag.all).append(count(i.id).as("num"))) + .changeFrom(_.prepend(tagFrom)) + .changeWhere(c => c && queryCondition(q)) + .groupBy(tag.tid) + .build + .query[TagCount] + .to[List] + } + + def searchCountSummary(q: Query): ConnectionIO[Int] = + findItemsBase(q, 0) + .withSelect(Nel.of(count(i.id).as("num"))) + .changeWhere(c => c && queryCondition(q)) + .build + .query[Int] + .unique + def findSelectedItems( q: Query, maxNoteLen: Int, diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala new file mode 100644 index 00000000..606ec6ff --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -0,0 +1,3 @@ +package docspell.store.queries + +case class SearchSummary(count: Int, tags: List[TagCount]) diff --git a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala new file mode 100644 index 00000000..e392f889 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala @@ -0,0 +1,5 @@ +package docspell.store.queries + +import docspell.store.records.RTag + +case class TagCount(tag: RTag, count: Int) From f3855628d5b8c449a4bc705d3bd396bb6e6365eb Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 23:08:15 +0100 Subject: [PATCH 27/38] Extend query builder with more functions --- .../scala/docspell/store/qb/DBFunction.scala | 10 ++++-- .../main/scala/docspell/store/qb/DSL.scala | 19 +++++++++-- .../main/scala/docspell/store/qb/Select.scala | 32 +++++++++++++++++-- .../store/qb/impl/DBFunctionBuilder.scala | 18 ++++++++--- .../docspell/store/queries/QCollective.scala | 2 +- 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 86ad7b1a..dc418810 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -13,9 +13,9 @@ object DBFunction { case class Count(column: Column[_]) extends DBFunction - case class Max(column: Column[_]) extends DBFunction + case class Max(expr: SelectExpr) extends DBFunction - case class Min(column: Column[_]) extends DBFunction + case class Min(expr: SelectExpr) extends DBFunction case class Coalesce(expr: SelectExpr, exprs: Vector[SelectExpr]) extends DBFunction @@ -25,6 +25,12 @@ object DBFunction { case class Substring(expr: SelectExpr, start: Int, length: Int) extends DBFunction + case class Cast(expr: SelectExpr, newType: String) extends DBFunction + + case class Avg(expr: SelectExpr) extends DBFunction + + case class Sum(expr: SelectExpr) extends DBFunction + sealed trait Operator object Operator { case object Plus extends Operator diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index fe3bda42..ce28d56e 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -68,11 +68,26 @@ trait DSL extends DoobieMeta { def countAll: DBFunction = DBFunction.CountAll + def max(e: SelectExpr): DBFunction = + DBFunction.Max(e) + def max(c: Column[_]): DBFunction = - DBFunction.Max(c) + max(c.s) + + def min(expr: SelectExpr): DBFunction = + DBFunction.Min(expr) def min(c: Column[_]): DBFunction = - DBFunction.Min(c) + min(c.s) + + def avg(expr: SelectExpr): DBFunction = + DBFunction.Avg(expr) + + def sum(expr: SelectExpr): DBFunction = + DBFunction.Sum(expr) + + def cast(expr: SelectExpr, targetType: String): DBFunction = + DBFunction.Cast(expr, targetType) def coalesce(expr: SelectExpr, more: SelectExpr*): DBFunction.Coalesce = DBFunction.Coalesce(expr, more.toVector) diff --git a/modules/store/src/main/scala/docspell/store/qb/Select.scala b/modules/store/src/main/scala/docspell/store/qb/Select.scala index d6c17113..d6150b18 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Select.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Select.scala @@ -29,8 +29,8 @@ sealed trait Select { def groupBy(gb: GroupBy): Select - def groupBy(c: Column[_]): Select = - groupBy(GroupBy(c)) + def groupBy(c: Column[_], cs: Column[_]*): Select = + groupBy(GroupBy(c, cs: _*)) def limit(batch: Batch): Select = this match { @@ -63,6 +63,8 @@ sealed trait Select { where(c.getOrElse(Condition.unit)) def where(c: Condition): Select + + def unwrap: Select.SimpleSelect } object Select { @@ -98,12 +100,18 @@ object Select { where: Condition, groupBy: Option[GroupBy] ) extends Select { + def unwrap: Select.SimpleSelect = + this + def groupBy(gb: GroupBy): SimpleSelect = copy(groupBy = Some(gb)) def distinct: SimpleSelect = copy(distinctFlag = true) + def noDistinct: SimpleSelect = + copy(distinctFlag = false) + def where(c: Condition): SimpleSelect = copy(where = c) @@ -119,11 +127,14 @@ object Select { def changeWhere(f: Condition => Condition): SimpleSelect = copy(where = f(where)) - def orderBy(ob: OrderBy, obs: OrderBy*): Select = + def orderBy(ob: OrderBy, obs: OrderBy*): Ordered = Ordered(this, ob, obs.toVector) } case class RawSelect(fragment: Fragment) extends Select { + def unwrap: Select.SimpleSelect = + sys.error("Cannot unwrap RawSelect") + def groupBy(gb: GroupBy): Select = sys.error("RawSelect doesn't support adding group by clause") @@ -147,6 +158,9 @@ object Select { } case class Union(q: Select, qs: Vector[Select]) extends Select { + def unwrap: Select.SimpleSelect = + q.unwrap + def groupBy(gb: GroupBy): Union = copy(q = q.groupBy(gb)) @@ -170,6 +184,9 @@ object Select { } case class Intersect(q: Select, qs: Vector[Select]) extends Select { + def unwrap: Select.SimpleSelect = + q.unwrap + def groupBy(gb: GroupBy): Intersect = copy(q = q.groupBy(gb)) @@ -194,6 +211,9 @@ object Select { case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy]) extends Select { + def unwrap: Select.SimpleSelect = + q.unwrap + def groupBy(gb: GroupBy): Ordered = copy(q = q.groupBy(gb)) @@ -215,6 +235,9 @@ object Select { } case class Limit(q: Select, batch: Batch) extends Select { + def unwrap: Select.SimpleSelect = + q.unwrap + def groupBy(gb: GroupBy): Limit = copy(q = q.groupBy(gb)) @@ -236,6 +259,9 @@ object Select { } case class WithCte(cte: CteBind, ctes: Vector[CteBind], q: Select) extends Select { + def unwrap: Select.SimpleSelect = + q.unwrap + def groupBy(gb: GroupBy): WithCte = copy(q = q.groupBy(gb)) diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index d99302e8..16c3e33f 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -16,11 +16,11 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.Count(col) => sql"COUNT(" ++ column(col) ++ fr")" - case DBFunction.Max(col) => - sql"MAX(" ++ column(col) ++ fr")" + case DBFunction.Max(expr) => + sql"MAX(" ++ SelectExprBuilder.build(expr) ++ fr")" - case DBFunction.Min(col) => - sql"MIN(" ++ column(col) ++ fr")" + case DBFunction.Min(expr) => + sql"MIN(" ++ SelectExprBuilder.build(expr) ++ fr")" case DBFunction.Coalesce(expr, exprs) => val v = exprs.prepended(expr).map(SelectExprBuilder.build) @@ -37,6 +37,16 @@ object DBFunctionBuilder extends CommonBuilder { buildOperator(op) ++ SelectExprBuilder.build(right) + case DBFunction.Cast(f, newType) => + sql"CAST(" ++ SelectExprBuilder.build(f) ++ + fr" AS" ++ Fragment.const(newType) ++ + sql")" + + case DBFunction.Avg(expr) => + sql"AVG(" ++ SelectExprBuilder.build(expr) ++ fr")" + + case DBFunction.Sum(expr) => + sql"SUM(" ++ SelectExprBuilder.build(expr) ++ fr")" } def buildOperator(op: DBFunction.Operator): Fragment = diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index 696d2294..b9fe40c7 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -86,7 +86,7 @@ object QCollective { select(t.all).append(count(ti.itemId).s), from(ti).innerJoin(t, ti.tagId === t.tid), t.cid === coll - ).group(GroupBy(t.name, t.tid, t.category)) + ).groupBy(t.name, t.tid, t.category) sql.build.query[TagCount].to[List] } From 77627534bcac1ab725074d300ca0228ac50b35e3 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 23:11:04 +0100 Subject: [PATCH 28/38] Improve on basic search summary --- .../docspell/backend/ops/OItemSearch.scala | 5 +- .../docspell/common/CustomFieldType.scala | 4 +- .../docspell/store/queries/FieldStats.scala | 17 +++++ .../scala/docspell/store/queries/QItem.scala | 70 ++++++++++++++++++- .../store/queries/SearchSummary.scala | 2 +- 5 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/FieldStats.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index fbfa69d2..9061b87a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -145,10 +145,7 @@ object OItemSearch { } def findItemsSummary(q: Query): F[SearchSummary] = - for { - tags <- store.transact(QItem.searchTagSummary(q)) - count <- store.transact(QItem.searchCountSummary(q)) - } yield SearchSummary(count, tags) + store.transact(QItem.searchStats(q)) def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store diff --git a/modules/common/src/main/scala/docspell/common/CustomFieldType.scala b/modules/common/src/main/scala/docspell/common/CustomFieldType.scala index 6c217550..55f9b6bf 100644 --- a/modules/common/src/main/scala/docspell/common/CustomFieldType.scala +++ b/modules/common/src/main/scala/docspell/common/CustomFieldType.scala @@ -2,6 +2,7 @@ package docspell.common import java.time.LocalDate +import cats.data.NonEmptyList import cats.implicits._ import io.circe._ @@ -92,7 +93,8 @@ object CustomFieldType { def bool: CustomFieldType = Bool def money: CustomFieldType = Money - val all: List[CustomFieldType] = List(Text, Numeric, Date, Bool, Money) + val all: NonEmptyList[CustomFieldType] = + NonEmptyList.of(Text, Numeric, Date, Bool, Money) def fromString(str: String): Either[String, CustomFieldType] = str.toLowerCase match { diff --git a/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala b/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala new file mode 100644 index 00000000..4fb90c50 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala @@ -0,0 +1,17 @@ +package docspell.store.queries + +import docspell.store.records.RCustomField + +case class FieldStats( + field: RCustomField, + count: Int, + avg: BigDecimal, + sum: BigDecimal, + max: BigDecimal, + min: BigDecimal +) + +object FieldStats { + def apply(field: RCustomField, count: Int): FieldStats = + FieldStats(field, count, BigDecimal(0), BigDecimal(0), BigDecimal(0), BigDecimal(0)) +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index fab67704..e5dbe4b9 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -15,7 +15,7 @@ import docspell.store.records._ import doobie.implicits._ import doobie.{Query => _, _} -import org.log4s._ +import org.log4s.getLogger object QItem { private[this] val logger = getLogger @@ -234,13 +234,20 @@ object QItem { sql.query[ListItem].stream } + def searchStats(q: Query): ConnectionIO[SearchSummary] = + for { + count <- searchCountSummary(q) + tags <- searchTagSummary(q) + fields <- searchFieldSummary(q) + } yield SearchSummary(count, tags, fields) + def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = from(ti) .innerJoin(tag, tag.tid === ti.tagId) .innerJoin(i, i.id === ti.itemId) - findItemsBase(q, 0) + findItemsBase(q, 0).unwrap .withSelect(select(tag.all).append(count(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) .changeWhere(c => c && queryCondition(q)) @@ -251,13 +258,70 @@ object QItem { } def searchCountSummary(q: Query): ConnectionIO[Int] = - findItemsBase(q, 0) + findItemsBase(q, 0).unwrap .withSelect(Nel.of(count(i.id).as("num"))) .changeWhere(c => c && queryCondition(q)) .build .query[Int] .unique + def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = { + val fieldJoin = + from(cv) + .innerJoin(cf, cf.id === cv.field) + .innerJoin(i, i.id === cv.itemId) + + val base = + findItemsBase(q, 0).unwrap + .changeFrom(_.prepend(fieldJoin)) + .changeWhere(c => c && queryCondition(q)) + .groupBy(GroupBy(cf.all)) + + val basicFields = Nel.of( + count(i.id).as("fc"), + lit(0).as("favg"), + lit(0).as("fsum"), + lit(0).as("fmax"), + lit(0).as("fmin") + ) + val valueNum = cast(cv.value.s, "decimal").s + val numericFields = Nel.of( + count(i.id).as("fc"), + avg(valueNum).as("favg"), + sum(valueNum).as("fsum"), + max(valueNum).as("fmax"), + min(valueNum).as("fmin") + ) + + val numTypes = Nel.of(CustomFieldType.money, CustomFieldType.numeric) + val query = + union( + base + .withSelect(select(cf.all).concatNel(basicFields)) + .changeWhere(c => c && cf.ftype.notIn(numTypes)), + base + .withSelect(select(cf.all).concatNel(numericFields)) + .changeWhere(c => c && cf.ftype.in(numTypes)) + ).build.query[FieldStats].to[List] + + val fallback = base + .withSelect(select(cf.all).concatNel(basicFields)) + .build + .query[FieldStats] + .to[List] + + query.attemptSql.flatMap { + case Right(res) => res.pure[ConnectionIO] + case Left(ex) => + Logger + .log4s[ConnectionIO](logger) + .error(ex)( + s"Calculating custom field summary failed. You may have invalid custom field values according to their type." + ) *> + fallback + } + } + def findSelectedItems( q: Query, maxNoteLen: Int, diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index 606ec6ff..36cd2a71 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -1,3 +1,3 @@ package docspell.store.queries -case class SearchSummary(count: Int, tags: List[TagCount]) +case class SearchSummary(count: Int, tags: List[TagCount], fields: List[FieldStats]) From 80e23d1c840761c40bfdf97efae784b3038006e5 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 23:33:03 +0100 Subject: [PATCH 29/38] Add a route to get search summary --- .../src/main/resources/docspell-openapi.yml | 83 +++++++++++++++++++ .../restserver/conv/Conversions.scala | 16 ++++ .../restserver/routes/ItemRoutes.scala | 8 ++ 3 files changed, 107 insertions(+) diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index cc70f5f0..91b6797d 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1375,6 +1375,27 @@ paths: schema: $ref: "#/components/schemas/ItemLightList" + /sec/item/searchStats: + post: + tags: [ Item ] + summary: Get basic statistics about the data of a search. + description: | + Takes a search query and returns a summary about the results. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemSearch" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" + /sec/item/{id}: get: tags: [ Item ] @@ -4146,6 +4167,23 @@ components: key: type: string format: ident + SearchStats: + description: | + A summary of search results. + required: + - count + - tagCloud + - fieldStats + properties: + count: + type: integer + format: int32 + tagCloud: + $ref: "#/components/schemas/TagCloud" + fieldStats: + type: array + items: + $ref: "#/components/schemas/FieldStats" ItemInsights: description: | Information about the items in docspell. @@ -4166,6 +4204,51 @@ components: format: int64 tagCloud: $ref: "#/components/schemas/TagCloud" + FieldStats: + description: | + Basic statistics about a custom field. + required: + - id + - name + - ftype + - count + - avg + - sum + - max + - min + properties: + id: + type: string + format: ident + name: + type: string + format: ident + label: + type: string + ftype: + type: string + format: customfieldtype + enum: + - text + - numeric + - date + - bool + - money + count: + type: integer + format: int32 + sum: + type: number + format: double + avg: + type: number + format: double + max: + type: number + format: double + min: + type: number + format: double TagCloud: description: | A tag "cloud" diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index f817fb10..6b8c2880 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -27,6 +27,22 @@ import org.log4s.Logger trait Conversions { + def mkSearchStats(sum: OItemSearch.SearchSummary): SearchStats = + SearchStats(sum.count, mkTagCloud(sum.tags), sum.fields.map(mkFieldStats)) + + def mkFieldStats(fs: docspell.store.queries.FieldStats): FieldStats = + FieldStats( + fs.field.id, + fs.field.name, + fs.field.label, + fs.field.ftype, + fs.count, + fs.sum.doubleValue, + fs.avg.doubleValue, + fs.max.doubleValue, + fs.min.doubleValue + ) + // insights def mkItemInsights(d: InsightData): ItemInsights = ItemInsights( diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index 9f39d180..eaa5bb7a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -143,6 +143,14 @@ object ItemRoutes { } } yield resp + case req @ POST -> Root / "searchStats" => + for { + mask <- req.as[ItemSearch] + query = Conversions.mkQuery(mask, user.account) + stats <- backend.itemSearch.findItemsSummary(query) + resp <- Ok(Conversions.mkSearchStats(stats)) + } yield resp + case GET -> Root / Ident(id) => for { item <- backend.itemSearch.findItem(id, user.account.collective) From a995ea8729d0c8829ffd69479276bad974038acd Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 16 Dec 2020 00:56:12 +0100 Subject: [PATCH 30/38] Update tag counts in search menu --- modules/webapp/src/main/elm/Api.elm | 12 ++++++ .../webapp/src/main/elm/Comp/SearchMenu.elm | 13 +++--- .../webapp/src/main/elm/Comp/TagSelect.elm | 40 +++++++++++++++++++ .../webapp/src/main/elm/Page/Home/Data.elm | 7 +++- .../webapp/src/main/elm/Page/Home/Update.elm | 7 ++++ 5 files changed, 72 insertions(+), 7 deletions(-) diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index c36c709f..d1241821 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -67,6 +67,7 @@ module Api exposing , itemDetail , itemIndexSearch , itemSearch + , itemSearchStats , login , loginSession , logout @@ -184,6 +185,7 @@ import Api.Model.ReferenceList exposing (ReferenceList) import Api.Model.Registration exposing (Registration) import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings) import Api.Model.ScanMailboxSettingsList exposing (ScanMailboxSettingsList) +import Api.Model.SearchStats exposing (SearchStats) import Api.Model.SentMails exposing (SentMails) import Api.Model.SimpleMail exposing (SimpleMail) import Api.Model.SourceAndTags exposing (SourceAndTags) @@ -1702,6 +1704,16 @@ itemSearch flags search receive = } +itemSearchStats : Flags -> ItemSearch -> (Result Http.Error SearchStats -> msg) -> Cmd msg +itemSearchStats flags search receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchStats" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.ItemSearch.encode search) + , expect = Http.expectJson receive Api.Model.SearchStats.decoder + } + + itemDetail : Flags -> String -> (Result Http.Error ItemDetail -> msg) -> Cmd msg itemDetail flags id receive = Http2.authGet diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index ab57fcbf..dd10279d 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -23,6 +23,7 @@ import Api.Model.IdName exposing (IdName) import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.PersonList exposing (PersonList) import Api.Model.ReferenceList exposing (ReferenceList) +import Api.Model.SearchStats exposing (SearchStats) import Api.Model.TagCloud exposing (TagCloud) import Comp.CustomFieldMultiInput import Comp.DatePicker @@ -338,7 +339,6 @@ type Msg | FromDueDateMsg Comp.DatePicker.Msg | UntilDueDateMsg Comp.DatePicker.Msg | ToggleInbox - | GetTagsResp (Result Http.Error TagCloud) | GetOrgResp (Result Http.Error ReferenceList) | GetEquipResp (Result Http.Error EquipmentList) | GetPersonResp (Result Http.Error PersonList) @@ -359,6 +359,7 @@ type Msg | SetTag String | CustomFieldMsg Comp.CustomFieldMultiInput.Msg | SetSource String + | GetStatsResp (Result Http.Error SearchStats) type alias NextState = @@ -425,7 +426,7 @@ updateDrop ddm flags settings msg model = { model = mdp , cmd = Cmd.batch - [ Api.getTagCloud flags GetTagsResp + [ Api.itemSearchStats flags (getItemSearch model) GetStatsResp , Api.getOrgLight flags GetOrgResp , Api.getEquipments flags "" GetEquipResp , Api.getPersons flags "" GetPersonResp @@ -475,12 +476,12 @@ updateDrop ddm flags settings msg model = SetTag id -> resetAndSet (TagSelectMsg (Comp.TagSelect.toggleTag id)) - GetTagsResp (Ok tags) -> + GetStatsResp (Ok stats) -> let selectModel = - List.sortBy .count tags.items + List.sortBy .count stats.tagCloud.items |> List.reverse - |> Comp.TagSelect.init model.tagSelection + |> Comp.TagSelect.modify model.tagSelection model.tagSelectModel model_ = { model | tagSelectModel = selectModel } @@ -491,7 +492,7 @@ updateDrop ddm flags settings msg model = , dragDrop = DD.DragDropData ddm Nothing } - GetTagsResp (Err _) -> + GetStatsResp (Err _) -> { model = model , cmd = Cmd.none , stateChange = False diff --git a/modules/webapp/src/main/elm/Comp/TagSelect.elm b/modules/webapp/src/main/elm/Comp/TagSelect.elm index 77f11b23..d72cb355 100644 --- a/modules/webapp/src/main/elm/Comp/TagSelect.elm +++ b/modules/webapp/src/main/elm/Comp/TagSelect.elm @@ -5,6 +5,7 @@ module Comp.TagSelect exposing , Selection , emptySelection , init + , modify , reset , toggleTag , update @@ -77,6 +78,45 @@ init sel tags = } +modify : Selection -> Model -> List TagCount -> Model +modify sel model tags = + let + newModel = + init sel tags + in + if List.isEmpty model.all then + newModel + + else + let + tagId t = + t.tag.id + + catId c = + c.name + + tagDict = + List.map (\e -> ( tagId e, e )) tags + |> Dict.fromList + + catDict = + List.map (\e -> ( catId e, e )) newModel.categories + |> Dict.fromList + + replaceTag e = + Dict.get e.tag.id tagDict |> Maybe.withDefault { e | count = 0 } + + replaceCat c = + Dict.get c.name catDict |> Maybe.withDefault { c | count = 0 } + in + { model + | all = List.map replaceTag model.all + , filteredTags = List.map replaceTag model.filteredTags + , categories = List.map replaceCat model.categories + , filteredCats = List.map replaceCat model.filteredCats + } + + reset : Model -> Model reset model = { model diff --git a/modules/webapp/src/main/elm/Page/Home/Data.elm b/modules/webapp/src/main/elm/Page/Home/Data.elm index 6c328b6d..3b4dfa01 100644 --- a/modules/webapp/src/main/elm/Page/Home/Data.elm +++ b/modules/webapp/src/main/elm/Page/Home/Data.elm @@ -19,6 +19,7 @@ module Page.Home.Data exposing import Api import Api.Model.BasicResult exposing (BasicResult) import Api.Model.ItemLightList exposing (ItemLightList) +import Api.Model.SearchStats exposing (SearchStats) import Browser.Dom as Dom import Comp.FixedDropdown import Comp.ItemCardList @@ -173,6 +174,7 @@ type Msg | DeleteAllResp (Result Http.Error BasicResult) | UiSettingsUpdated | SetLinkTarget LinkTarget + | SearchStatsResp (Result Http.Error SearchStats) type SearchType @@ -237,7 +239,10 @@ doSearchDefaultCmd param model = } in if param.offset == 0 then - Api.itemSearch param.flags mask (ItemSearchResp param.scroll) + Cmd.batch + [ Api.itemSearch param.flags mask (ItemSearchResp param.scroll) + , Api.itemSearchStats param.flags mask SearchStatsResp + ] else Api.itemSearch param.flags mask ItemSearchAddResp diff --git a/modules/webapp/src/main/elm/Page/Home/Update.elm b/modules/webapp/src/main/elm/Page/Home/Update.elm index 6b01115f..70168d24 100644 --- a/modules/webapp/src/main/elm/Page/Home/Update.elm +++ b/modules/webapp/src/main/elm/Page/Home/Update.elm @@ -550,6 +550,13 @@ update mId key flags settings msg model = in update mId key flags settings (DoSearch model.lastSearchType) model_ + SearchStatsResp result -> + let + lm = + SearchMenuMsg (Comp.SearchMenu.GetStatsResp result) + in + update mId key flags settings lm model + --- Helpers From 8fba637ebe02359fb0b5bd4340b22f9e7389f6d8 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 16 Dec 2020 01:14:27 +0100 Subject: [PATCH 31/38] Add folder counts to search summary --- .../src/main/resources/docspell-openapi.yml | 24 +++++++++++++++++++ .../restserver/conv/Conversions.scala | 10 +++++++- .../docspell/store/queries/FolderCount.scala | 5 ++++ .../scala/docspell/store/queries/QItem.scala | 21 ++++++++++++---- .../store/queries/SearchSummary.scala | 7 +++++- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/FolderCount.scala diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 91b6797d..9f70659c 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -4174,6 +4174,7 @@ components: - count - tagCloud - fieldStats + - folderStats properties: count: type: integer @@ -4184,6 +4185,10 @@ components: type: array items: $ref: "#/components/schemas/FieldStats" + folderStats: + type: array + items: + $ref: "#/components/schemas/FolderStats" ItemInsights: description: | Information about the items in docspell. @@ -4204,6 +4209,25 @@ components: format: int64 tagCloud: $ref: "#/components/schemas/TagCloud" + FolderStats: + description: | + Count of folder usage. + required: + - id + - name + - owner + - count + properties: + id: + type: string + format: ident + name: + type: string + owner: + $ref: "#/components/schemas/IdName" + count: + type: integer + format: int32 FieldStats: description: | Basic statistics about a custom field. diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 6b8c2880..eefaa9cb 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -28,7 +28,15 @@ import org.log4s.Logger trait Conversions { def mkSearchStats(sum: OItemSearch.SearchSummary): SearchStats = - SearchStats(sum.count, mkTagCloud(sum.tags), sum.fields.map(mkFieldStats)) + SearchStats( + sum.count, + mkTagCloud(sum.tags), + sum.fields.map(mkFieldStats), + sum.folders.map(mkFolderStats) + ) + + def mkFolderStats(fs: docspell.store.queries.FolderCount): FolderStats = + FolderStats(fs.id, fs.name, mkIdName(fs.owner), fs.count) def mkFieldStats(fs: docspell.store.queries.FieldStats): FieldStats = FieldStats( diff --git a/modules/store/src/main/scala/docspell/store/queries/FolderCount.scala b/modules/store/src/main/scala/docspell/store/queries/FolderCount.scala new file mode 100644 index 00000000..a1f93c55 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/FolderCount.scala @@ -0,0 +1,5 @@ +package docspell.store.queries + +import docspell.common._ + +case class FolderCount(id: Ident, name: String, owner: IdRef, count: Int) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index e5dbe4b9..b0184aaf 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -236,10 +236,11 @@ object QItem { def searchStats(q: Query): ConnectionIO[SearchSummary] = for { - count <- searchCountSummary(q) - tags <- searchTagSummary(q) - fields <- searchFieldSummary(q) - } yield SearchSummary(count, tags, fields) + count <- searchCountSummary(q) + tags <- searchTagSummary(q) + fields <- searchFieldSummary(q) + folders <- searchFolderSummary(q) + } yield SearchSummary(count, tags, fields, folders) def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = @@ -265,6 +266,18 @@ object QItem { .query[Int] .unique + def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = { + val fu = RUser.as("fu") + findItemsBase(q, 0).unwrap + .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num"))) + .changeFrom(_.innerJoin(fu, fu.uid === f.owner)) + .changeWhere(c => c && queryCondition(q)) + .groupBy(f.id, f.name, f.owner, fu.login) + .build + .query[FolderCount] + .to[List] + } + def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = { val fieldJoin = from(cv) diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index 36cd2a71..0530c211 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -1,3 +1,8 @@ package docspell.store.queries -case class SearchSummary(count: Int, tags: List[TagCount], fields: List[FieldStats]) +case class SearchSummary( + count: Int, + tags: List[TagCount], + fields: List[FieldStats], + folders: List[FolderCount] +) From b66738b4c3558134703b73a19d6e96e54812ebb8 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 16 Dec 2020 19:20:30 +0100 Subject: [PATCH 32/38] Add folder count to search menu --- .../webapp/src/main/elm/Comp/FolderSelect.elm | 50 +++++++++++++------ .../webapp/src/main/elm/Comp/SearchMenu.elm | 35 ++++--------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/FolderSelect.elm b/modules/webapp/src/main/elm/Comp/FolderSelect.elm index 4e516756..5cf80bd6 100644 --- a/modules/webapp/src/main/elm/Comp/FolderSelect.elm +++ b/modules/webapp/src/main/elm/Comp/FolderSelect.elm @@ -3,6 +3,7 @@ module Comp.FolderSelect exposing , Msg , deselect , init + , modify , setSelected , update , updateDrop @@ -10,7 +11,8 @@ module Comp.FolderSelect exposing , viewDrop ) -import Api.Model.FolderItem exposing (FolderItem) +import Api.Model.FolderStats exposing (FolderStats) +import Dict import Html exposing (..) import Html.Attributes exposing (..) import Html.Events exposing (onClick) @@ -20,13 +22,13 @@ import Util.List type alias Model = - { all : List FolderItem + { all : List FolderStats , selected : Maybe String , expanded : Bool } -init : Maybe FolderItem -> List FolderItem -> Model +init : Maybe FolderStats -> List FolderStats -> Model init selected all = { all = List.sortBy .name all , selected = Maybe.map .id selected @@ -34,6 +36,26 @@ init selected all = } +modify : Maybe FolderStats -> Model -> List FolderStats -> Model +modify selected model all = + if List.isEmpty model.all then + init selected all + + else + let + folderDict = + List.map (\f -> ( f.id, f )) all + |> Dict.fromList + + replaced el = + Dict.get el.id folderDict |> Maybe.withDefault { el | count = 0 } + in + { model + | all = List.map replaced model.all + , selected = Maybe.map .id selected + } + + setSelected : String -> Model -> Maybe Msg setSelected id model = List.filter (\fi -> fi.id == id) model.all @@ -43,12 +65,7 @@ setSelected id model = deselect : Model -> Maybe Msg deselect model = - case model.selected of - Just id -> - setSelected id model - - Nothing -> - Nothing + Maybe.andThen (\id -> setSelected id model) model.selected @@ -56,12 +73,12 @@ deselect model = type Msg - = Toggle FolderItem + = Toggle FolderStats | ToggleExpand | FolderDDMsg DD.Msg -update : Msg -> Model -> ( Model, Maybe FolderItem ) +update : Msg -> Model -> ( Model, Maybe FolderStats ) update msg model = let ( m, f, _ ) = @@ -74,7 +91,7 @@ updateDrop : DD.Model -> Msg -> Model - -> ( Model, Maybe FolderItem, DD.DragDropData ) + -> ( Model, Maybe FolderStats, DD.DragDropData ) updateDrop dropModel msg model = case msg of Toggle item -> @@ -105,7 +122,7 @@ updateDrop dropModel msg model = ( model, selectedFolder model, ddd ) -selectedFolder : Model -> Maybe FolderItem +selectedFolder : Model -> Maybe FolderStats selectedFolder model = let isSelected f = @@ -177,7 +194,7 @@ collapseToggle max model = ToggleExpand -viewItem : DD.Model -> Model -> FolderItem -> Html Msg +viewItem : DD.Model -> Model -> FolderStats -> Html Msg viewItem dropModel model item = let selected = @@ -206,8 +223,11 @@ viewItem dropModel model item = ) [ i [ class icon ] [] , div [ class "content" ] - [ div [ class "header" ] + [ div [ class "description" ] [ text item.name + , div [ class "ui right floated circular label" ] + [ text (String.fromInt item.count) + ] ] ] ] diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index dd10279d..3d023ed1 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -19,6 +19,7 @@ import Api.Model.Equipment exposing (Equipment) import Api.Model.EquipmentList exposing (EquipmentList) import Api.Model.FolderItem exposing (FolderItem) import Api.Model.FolderList exposing (FolderList) +import Api.Model.FolderStats exposing (FolderStats) import Api.Model.IdName exposing (IdName) import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.PersonList exposing (PersonList) @@ -60,7 +61,7 @@ type alias Model = , concPersonModel : Comp.Dropdown.Model IdName , concEquipmentModel : Comp.Dropdown.Model Equipment , folderList : Comp.FolderSelect.Model - , selectedFolder : Maybe FolderItem + , selectedFolder : Maybe FolderStats , inboxCheckbox : Bool , fromDateModel : DatePicker , fromDate : Maybe Int @@ -350,7 +351,6 @@ type Msg | ResetForm | KeyUpMsg (Maybe KeyCode) | FolderSelectMsg Comp.FolderSelect.Msg - | GetFolderResp (Result Http.Error FolderList) | SetCorrOrg IdName | SetCorrPerson IdName | SetConcPerson IdName @@ -430,7 +430,6 @@ updateDrop ddm flags settings msg model = , Api.getOrgLight flags GetOrgResp , Api.getEquipments flags "" GetEquipResp , Api.getPersons flags "" GetPersonResp - , Api.getFolders flags "" False GetFolderResp , Cmd.map CustomFieldMsg (Comp.CustomFieldMultiInput.initCmd flags) , cdp ] @@ -484,7 +483,13 @@ updateDrop ddm flags settings msg model = |> Comp.TagSelect.modify model.tagSelection model.tagSelectModel model_ = - { model | tagSelectModel = selectModel } + { model + | tagSelectModel = selectModel + , folderList = + Comp.FolderSelect.modify model.selectedFolder + model.folderList + stats.folderStats + } in { model = model_ , cmd = Cmd.none @@ -797,28 +802,6 @@ updateDrop ddm flags settings msg model = , dragDrop = DD.DragDropData ddm Nothing } - GetFolderResp (Ok fs) -> - let - model_ = - { model - | folderList = - Util.Folder.onlyVisible flags fs.items - |> Comp.FolderSelect.init model.selectedFolder - } - in - { model = model_ - , cmd = Cmd.none - , stateChange = False - , dragDrop = DD.DragDropData ddm Nothing - } - - GetFolderResp (Err _) -> - { model = model - , cmd = Cmd.none - , stateChange = False - , dragDrop = DD.DragDropData ddm Nothing - } - FolderSelectMsg lm -> let ( fsm, sel, ddd ) = From 8d7b3c7d746f24fa6d33531a160e7eac8157bf72 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 16 Dec 2020 21:34:53 +0100 Subject: [PATCH 33/38] Show custom field summary above results --- modules/webapp/src/main/elm/Data/Items.elm | 2 +- .../webapp/src/main/elm/Page/Home/Data.elm | 2 + .../webapp/src/main/elm/Page/Home/Update.elm | 5 +- .../webapp/src/main/elm/Page/Home/View.elm | 101 +++++++++++++++--- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/modules/webapp/src/main/elm/Data/Items.elm b/modules/webapp/src/main/elm/Data/Items.elm index 40a0e748..fbb9c9a3 100644 --- a/modules/webapp/src/main/elm/Data/Items.elm +++ b/modules/webapp/src/main/elm/Data/Items.elm @@ -45,7 +45,7 @@ concat l0 l1 = suff = List.drop 1 l1.groups in - ItemLightList (prev ++ [ ng ] ++ suff) + ItemLightList (prev ++ (ng :: suff)) else ItemLightList (l0.groups ++ l1.groups) diff --git a/modules/webapp/src/main/elm/Page/Home/Data.elm b/modules/webapp/src/main/elm/Page/Home/Data.elm index 3b4dfa01..e30b1dec 100644 --- a/modules/webapp/src/main/elm/Page/Home/Data.elm +++ b/modules/webapp/src/main/elm/Page/Home/Data.elm @@ -53,6 +53,7 @@ type alias Model = , lastSearchType : SearchType , dragDropData : DD.DragDropData , scrollToCard : Maybe String + , searchStats : SearchStats } @@ -117,6 +118,7 @@ init flags viewMode = DD.DragDropData DD.init Nothing , scrollToCard = Nothing , viewMode = viewMode + , searchStats = Api.Model.SearchStats.empty } diff --git a/modules/webapp/src/main/elm/Page/Home/Update.elm b/modules/webapp/src/main/elm/Page/Home/Update.elm index 70168d24..ec8c7f68 100644 --- a/modules/webapp/src/main/elm/Page/Home/Update.elm +++ b/modules/webapp/src/main/elm/Page/Home/Update.elm @@ -554,8 +554,11 @@ update mId key flags settings msg model = let lm = SearchMenuMsg (Comp.SearchMenu.GetStatsResp result) + + stats = + Result.withDefault model.searchStats result in - update mId key flags settings lm model + update mId key flags settings lm { model | searchStats = stats } diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index b59fcb8e..0f46c3bf 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -7,7 +7,9 @@ import Comp.ItemDetail.EditMenu import Comp.SearchMenu import Comp.YesNoDimmer import Data.Flags exposing (Flags) +import Data.Icons as Icons import Data.ItemSelection +import Data.Money import Data.UiSettings exposing (UiSettings) import Html exposing (..) import Html.Attributes exposing (..) @@ -110,20 +112,25 @@ view flags settings model = , ( "item-card-list", True ) ] ] - [ viewBar flags model - , case model.viewMode of - SelectView svm -> - Html.map DeleteSelectedConfirmMsg - (Comp.YesNoDimmer.view2 (selectAction == DeleteSelected) - deleteAllDimmer - svm.deleteAllConfirm - ) + (List.concat + [ viewBar flags model + , case model.viewMode of + SelectView svm -> + [ Html.map DeleteSelectedConfirmMsg + (Comp.YesNoDimmer.view2 (selectAction == DeleteSelected) + deleteAllDimmer + svm.deleteAllConfirm + ) + ] - _ -> - span [ class "invisible" ] [] - , Html.map ItemCardListMsg - (Comp.ItemCardList.view itemViewCfg settings model.itemListModel) - ] + _ -> + [] + , viewStats flags model + , [ Html.map ItemCardListMsg + (Comp.ItemCardList.view itemViewCfg settings model.itemListModel) + ] + ] + ) , div [ classList [ ( "sixteen wide column", True ) @@ -157,6 +164,66 @@ view flags settings model = ] +viewStats : Flags -> Model -> List (Html Msg) +viewStats _ model = + let + stats = + model.searchStats + + isNumField f = + f.sum > 0 + + statValues f = + tr [ class "center aligned" ] + [ td [ class "left aligned" ] + [ div [ class "ui basic label" ] + [ Icons.customFieldTypeIconString "" f.ftype + , text (Maybe.withDefault f.name f.label) + ] + ] + , td [] + [ f.count |> String.fromInt |> text + ] + , td [] + [ f.sum |> Data.Money.format |> text + ] + , td [] + [ f.avg |> Data.Money.format |> text + ] + , td [] + [ f.min |> Data.Money.format |> text + ] + , td [] + [ f.max |> Data.Money.format |> text + ] + ] + + fields = + List.filter isNumField stats.fieldStats + in + if List.isEmpty fields then + [] + + else + [ div [ class "ui container" ] + [ table [ class "ui very basic tiny six column table" ] + [ thead [] + [ tr [ class "center aligned" ] + [ th [] [] + , th [] [ text "Count" ] + , th [] [ text "Sum" ] + , th [] [ text "Avg" ] + , th [] [ text "Min" ] + , th [] [ text "Max" ] + ] + ] + , tbody [] + (List.map statValues fields) + ] + ] + ] + + viewLeftMenu : Flags -> UiSettings -> Model -> List (Html Msg) viewLeftMenu flags settings model = let @@ -206,17 +273,17 @@ viewLeftMenu flags settings model = searchMenu -viewBar : Flags -> Model -> Html Msg +viewBar : Flags -> Model -> List (Html Msg) viewBar flags model = case model.viewMode of SimpleView -> - viewSearchBar flags model + [ viewSearchBar flags model ] SearchView -> - div [ class "hidden invisible" ] [] + [] SelectView svm -> - viewActionBar flags svm model + [ viewActionBar flags svm model ] viewActionBar : Flags -> SelectViewModel -> Model -> Html Msg From 6346bf6a344a3dddce7cb41ed56d71cdf240a9d8 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 00:11:33 +0100 Subject: [PATCH 34/38] Add summary for fulltext searches --- build.sbt | 6 +- .../docspell/backend/ops/OFulltext.scala | 57 +++++++++++++++++++ .../restserver/routes/ItemRoutes.scala | 15 ++++- .../webapp/src/main/elm/Page/Home/View.elm | 42 ++++++++------ 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index 02f2401b..ff0f60c5 100644 --- a/build.sbt +++ b/build.sbt @@ -445,7 +445,8 @@ val joex = project buildInfoPackage := "docspell.joex", reStart / javaOptions ++= Seq( s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" - ) + ), + Revolver.enableDebugging(port = 5051, suspend = false) ) .dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) @@ -492,7 +493,8 @@ val restserver = project ), reStart / javaOptions ++= Seq( s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" - ) + ), + Revolver.enableDebugging(port = 5050, suspend = false) ) .dependsOn(restapi, joexapi, backend, webapp, ftssolr) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index d7f416d2..52e23571 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -7,12 +7,15 @@ import fs2.Stream import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ +import docspell.common.syntax.all._ import docspell.ftsclient._ import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob import docspell.store.{Store, qb} +import org.log4s.getLogger + trait OFulltext[F[_]] { def findItems(maxNoteLen: Int)( @@ -34,6 +37,9 @@ trait OFulltext[F[_]] { batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] + def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary] + def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary] + /** Clears the full-text index completely and launches a task that * indexes all data. */ @@ -46,6 +52,7 @@ trait OFulltext[F[_]] { } object OFulltext { + private[this] val logger = getLogger case class FtsInput( query: String, @@ -77,12 +84,14 @@ object OFulltext { Resource.pure[F, OFulltext[F]](new OFulltext[F] { def reindexAll: F[Unit] = for { + _ <- logger.finfo(s"Re-index all.") job <- JobFactory.reIndexAll[F] _ <- queue.insertIfNew(job) *> joex.notifyAllNodes } yield () def reindexCollective(account: AccountId): F[Unit] = for { + _ <- logger.fdebug(s"Re-index collective: $account") exist <- store.transact( RJob.findNonFinalByTracker(DocspellSystem.migrationTaskTracker) ) @@ -107,6 +116,7 @@ object OFulltext { FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost) ) for { + _ <- logger.ftrace(s"Find index only: ${ftsQ.query}/${batch}") folders <- store.transact(QFolder.getMemberFolders(account)) ftsR <- fts.search(fq.withFolders(folders)) ftsItems = ftsR.results.groupBy(_.itemId) @@ -133,6 +143,32 @@ object OFulltext { } yield res } + def findIndexOnlySummary( + account: AccountId, + ftsQ: OFulltext.FtsInput + ): F[SearchSummary] = { + val fq = FtsQuery( + ftsQ.query, + account.collective, + Set.empty, + Set.empty, + 500, + 0, + FtsQuery.HighlightSetting.default + ) + + for { + folder <- store.transact(QFolder.getMemberFolders(account)) + itemIds <- fts + .searchAll(fq.withFolders(folder)) + .flatMap(r => Stream.emits(r.results.map(_.itemId))) + .compile + .to(Set) + q = Query.empty(account).copy(itemIds = itemIds.some) + res <- store.transact(QItem.searchStats(q)) + } yield res + } + def findItems( maxNoteLen: Int )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] = @@ -167,6 +203,27 @@ object OFulltext { .compile .toVector + def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] = + for { + search <- itemSearch.findItems(0)(q, Batch.all) + fq = FtsQuery( + ftsQ.query, + q.account.collective, + search.map(_.id).toSet, + Set.empty, + 500, + 0, + FtsQuery.HighlightSetting.default + ) + items <- fts + .searchAll(fq) + .flatMap(r => Stream.emits(r.results.map(_.itemId))) + .compile + .to(Set) + qnext = q.copy(itemIds = items.some) + res <- store.transact(QItem.searchStats(qnext)) + } yield res + // Helper private def findItemsFts[A: ItemId, B]( diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index eaa5bb7a..e0b0c5b8 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -147,8 +147,19 @@ object ItemRoutes { for { mask <- req.as[ItemSearch] query = Conversions.mkQuery(mask, user.account) - stats <- backend.itemSearch.findItemsSummary(query) - resp <- Ok(Conversions.mkSearchStats(stats)) + stats <- mask match { + case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => + logger.finfo(s"Make index only summary: $ftq") *> + backend.fulltext.findIndexOnlySummary( + user.account, + OFulltext.FtsInput(ftq.query) + ) + case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => + backend.fulltext.findItemsSummary(query, OFulltext.FtsInput(fq)) + case _ => + backend.itemSearch.findItemsSummary(query) + } + resp <- Ok(Conversions.mkSearchStats(stats)) } yield resp case GET -> Root / Ident(id) => diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index 0f46c3bf..b4c93634 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -201,27 +201,37 @@ viewStats _ model = fields = List.filter isNumField stats.fieldStats in - if List.isEmpty fields then - [] - - else - [ div [ class "ui container" ] - [ table [ class "ui very basic tiny six column table" ] - [ thead [] - [ tr [ class "center aligned" ] - [ th [] [] - , th [] [ text "Count" ] - , th [] [ text "Sum" ] - , th [] [ text "Avg" ] - , th [] [ text "Min" ] - , th [] [ text "Max" ] + [ div [ class "ui container" ] + [ div [ class "ui middle aligned grid" ] + [ div [ class "three wide center aligned column" ] + [ div [ class "ui small statistic" ] + [ div [ class "value" ] + [ String.fromInt stats.count |> text + ] + , div [ class "label" ] + [ text "Results" ] ] - , tbody [] - (List.map statValues fields) + ] + , div [ class "thirteen wide column" ] + [ table [ class "ui very basic tiny six column table" ] + [ thead [] + [ tr [ class "center aligned" ] + [ th [] [] + , th [] [ text "Count" ] + , th [] [ text "Sum" ] + , th [] [ text "Avg" ] + , th [] [ text "Min" ] + , th [] [ text "Max" ] + ] + ] + , tbody [] + (List.map statValues fields) + ] ] ] ] + ] viewLeftMenu : Flags -> UiSettings -> Model -> List (Html Msg) From c9d4e8ec460e7d59b9f963ff6fe506a1ff4d75aa Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 21:02:23 +0100 Subject: [PATCH 35/38] Fix selecting items in multi-edit mode For some to me unknown reason, changing the dom slightly (removing hidden elements), resulted in a different event dispatching. The cards while being attached to an event would reload the page as if the event is propagated. This happned by commit #8d7b3c7d in Home/View.elm. Adding the hidden nodes back into the dom, "fixed" it. This change now gives a better fix in assuring that every anchor has either a sensible `href` or an event and a `href #`. --- modules/webapp/src/main/elm/Comp/ItemCard.elm | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/ItemCard.elm b/modules/webapp/src/main/elm/Comp/ItemCard.elm index 0de41d9e..e5eb1295 100644 --- a/modules/webapp/src/main/elm/Comp/ItemCard.elm +++ b/modules/webapp/src/main/elm/Comp/ItemCard.elm @@ -148,10 +148,13 @@ view cfg settings model item = cardAction = case cfg.selection of Data.ItemSelection.Inactive -> - Page.href (ItemDetailPage item.id) + [ Page.href (ItemDetailPage item.id) + ] Data.ItemSelection.Active ids -> - onClick (ToggleSelectItem ids item.id) + [ onClick (ToggleSelectItem ids item.id) + , href "#" + ] selectedDimmer = div @@ -162,8 +165,7 @@ view cfg settings model item = ] [ div [ class "content" ] [ a - [ cardAction - ] + cardAction [ i [ class "huge icons purple" ] [ i [ class "big circle outline icon" ] [] , i [ class "check icon" ] [] @@ -311,7 +313,7 @@ notesContent settings item = ] -mainContent : Attribute Msg -> String -> Bool -> UiSettings -> ViewConfig -> ItemLight -> Html Msg +mainContent : List (Attribute Msg) -> String -> Bool -> UiSettings -> ViewConfig -> ItemLight -> Html Msg mainContent cardAction cardColor isConfirmed settings _ item = let dirIcon = @@ -327,10 +329,7 @@ mainContent cardAction cardColor isConfirmed settings _ item = settings.cardSubtitleTemplate.template in a - [ class "content" - , href "#" - , cardAction - ] + (class "content" :: cardAction) [ if fieldHidden Data.Fields.Direction then div [ class "header" ] [ IT.render titlePattern item |> text @@ -419,7 +418,7 @@ mainTagsAndFields settings item = (renderFields ++ renderTags) -previewImage : UiSettings -> Attribute Msg -> Model -> ItemLight -> Html Msg +previewImage : UiSettings -> List (Attribute Msg) -> Model -> ItemLight -> Html Msg previewImage settings cardAction model item = let mainAttach = @@ -431,10 +430,11 @@ previewImage settings cardAction model item = |> Maybe.withDefault (Api.itemBasePreviewURL item.id) in a - [ class "image ds-card-image" - , Data.UiSettings.cardPreviewSize settings - , cardAction - ] + ([ class "image ds-card-image" + , Data.UiSettings.cardPreviewSize settings + ] + ++ cardAction + ) [ img [ class "preview-image" , src previewUrl From 69f57d1eb1f453a749f4bddfe9b449d73ca1efa4 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 21:15:33 +0100 Subject: [PATCH 36/38] Replace empty hrefs with a href # --- modules/webapp/src/main/elm/App/View.elm | 2 +- modules/webapp/src/main/elm/Comp/ContactField.elm | 2 +- modules/webapp/src/main/elm/Comp/Dropzone.elm | 2 +- modules/webapp/src/main/elm/Comp/EmailSettingsManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/EquipmentManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/ItemDetail/EditMenu.elm | 4 ++-- modules/webapp/src/main/elm/Comp/ItemDetail/View.elm | 8 ++++---- modules/webapp/src/main/elm/Comp/ItemList.elm | 6 +++--- modules/webapp/src/main/elm/Comp/OrgManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/PersonManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/SourceManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/TagManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/UserManage.elm | 4 ++-- modules/webapp/src/main/elm/Comp/YesNoDimmer.elm | 4 ++-- modules/webapp/src/main/elm/Page/Home/View.elm | 2 +- modules/webapp/src/main/elm/Page/NewInvite/View.elm | 2 +- modules/webapp/src/main/elm/Page/Upload/View.elm | 6 +++--- 18 files changed, 35 insertions(+), 35 deletions(-) diff --git a/modules/webapp/src/main/elm/App/View.elm b/modules/webapp/src/main/elm/App/View.elm index 63e4839f..8ff8e890 100644 --- a/modules/webapp/src/main/elm/App/View.elm +++ b/modules/webapp/src/main/elm/App/View.elm @@ -230,7 +230,7 @@ loginInfo model = , div [ class "divider" ] [] , a [ class "icon item" - , href "" + , href "#" , onClick Logout ] [ i [ class "sign out icon" ] [] diff --git a/modules/webapp/src/main/elm/Comp/ContactField.elm b/modules/webapp/src/main/elm/Comp/ContactField.elm index f93c34ac..cc283783 100644 --- a/modules/webapp/src/main/elm/Comp/ContactField.elm +++ b/modules/webapp/src/main/elm/Comp/ContactField.elm @@ -135,7 +135,7 @@ view1 settings compact model = , value model.value ] [] - , a [ class "ui button", onClick AddContact, href "" ] + , a [ class "ui button", onClick AddContact, href "#" ] [ text "Add" ] ] diff --git a/modules/webapp/src/main/elm/Comp/Dropzone.elm b/modules/webapp/src/main/elm/Comp/Dropzone.elm index b34381ec..8c650593 100644 --- a/modules/webapp/src/main/elm/Comp/Dropzone.elm +++ b/modules/webapp/src/main/elm/Comp/Dropzone.elm @@ -129,7 +129,7 @@ view model = , ( "disabled", not model.state.active ) ] , onClick PickFiles - , href "" + , href "#" ] [ i [ class "folder open icon" ] [] , text "Select ..." diff --git a/modules/webapp/src/main/elm/Comp/EmailSettingsManage.elm b/modules/webapp/src/main/elm/Comp/EmailSettingsManage.elm index 953e2c7c..100d92d4 100644 --- a/modules/webapp/src/main/elm/Comp/EmailSettingsManage.elm +++ b/modules/webapp/src/main/elm/Comp/EmailSettingsManage.elm @@ -268,12 +268,12 @@ viewForm settings model = , a [ class "ui secondary button" , onClick (SetViewMode Table) - , href "" + , href "#" ] [ text "Cancel" ] , if model.formModel.settings.name /= "" then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/EquipmentManage.elm b/modules/webapp/src/main/elm/Comp/EquipmentManage.elm index 9542f518..8bf4e1a9 100644 --- a/modules/webapp/src/main/elm/Comp/EquipmentManage.elm +++ b/modules/webapp/src/main/elm/Comp/EquipmentManage.elm @@ -282,11 +282,11 @@ viewForm model = , button [ class "ui primary button", type_ "submit" ] [ text "Submit" ] - , a [ class "ui secondary button", onClick (SetViewMode Table), href "" ] + , a [ class "ui secondary button", onClick (SetViewMode Table), href "#" ] [ text "Cancel" ] , if not newEquipment then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm b/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm index 7536e91f..bb33e10e 100644 --- a/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm +++ b/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm @@ -268,12 +268,12 @@ viewForm settings model = , a [ class "ui secondary button" , onClick (SetViewMode Table) - , href "" + , href "#" ] [ text "Cancel" ] , if model.formModel.settings.name /= "" then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/EditMenu.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/EditMenu.elm index 82578a52..8bd70dd0 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/EditMenu.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/EditMenu.elm @@ -772,7 +772,7 @@ item visible. This message will disappear then. actionInputDatePicker model.itemDatePicker ) - , a [ class "ui icon button", href "", onClick RemoveDate ] + , a [ class "ui icon button", href "#", onClick RemoveDate ] [ i [ class "trash alternate outline icon" ] [] ] , Icons.dateIcon "" @@ -791,7 +791,7 @@ item visible. This message will disappear then. actionInputDatePicker model.dueDatePicker ) - , a [ class "ui icon button", href "", onClick RemoveDueDate ] + , a [ class "ui icon button", href "#", onClick RemoveDueDate ] [ i [ class "trash alternate outline icon" ] [] ] , Icons.dueDateIcon "" ] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm index 2529b6b7..b541f20b 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/View.elm @@ -155,7 +155,7 @@ renderDetailMenu settings inav model = ] , title "Edit Metadata" , onClick ToggleMenu - , href "" + , href "#" ] [ i [ class "edit icon" ] [] ] @@ -835,7 +835,7 @@ item visible. This message will disappear then. actionInputDatePicker model.itemDatePicker ) - , a [ class "ui icon button", href "", onClick RemoveDate ] + , a [ class "ui icon button", href "#", onClick RemoveDate ] [ i [ class "trash alternate outline icon" ] [] ] , Icons.dateIcon "" @@ -855,7 +855,7 @@ item visible. This message will disappear then. actionInputDatePicker model.dueDatePicker ) - , a [ class "ui icon button", href "", onClick RemoveDueDate ] + , a [ class "ui icon button", href "#", onClick RemoveDueDate ] [ i [ class "trash alternate outline icon" ] [] ] , Icons.dueDateIcon "" ] @@ -957,7 +957,7 @@ renderSuggestions model mkName idnames tagger = , div [ class "menu" ] <| (idnames |> List.take 5 - |> List.map (\p -> a [ class "item", href "", onClick (tagger p) ] [ text (mkName p) ]) + |> List.map (\p -> a [ class "item", href "#", onClick (tagger p) ] [ text (mkName p) ]) ) ] ] diff --git a/modules/webapp/src/main/elm/Comp/ItemList.elm b/modules/webapp/src/main/elm/Comp/ItemList.elm index a6876184..dc1dee5f 100644 --- a/modules/webapp/src/main/elm/Comp/ItemList.elm +++ b/modules/webapp/src/main/elm/Comp/ItemList.elm @@ -108,7 +108,7 @@ view model = [ class "item" , title "Expand all" , onClick ExpandAll - , href "" + , href "#" ] [ i [ class "double angle down icon" ] [] ] @@ -116,7 +116,7 @@ view model = [ class "item" , title "Collapse all" , onClick CollapseAll - , href "" + , href "#" ] [ i [ class "double angle up icon" ] [] ] @@ -166,7 +166,7 @@ viewGroup model group = , a [ class "header" , onClick (ToggleGroupState group) - , href "" + , href "#" ] [ text group.name ] diff --git a/modules/webapp/src/main/elm/Comp/OrgManage.elm b/modules/webapp/src/main/elm/Comp/OrgManage.elm index ad56269d..5bd7659b 100644 --- a/modules/webapp/src/main/elm/Comp/OrgManage.elm +++ b/modules/webapp/src/main/elm/Comp/OrgManage.elm @@ -283,11 +283,11 @@ viewForm settings model = , button [ class "ui primary button", type_ "submit" ] [ text "Submit" ] - , a [ class "ui secondary button", onClick (SetViewMode Table), href "" ] + , a [ class "ui secondary button", onClick (SetViewMode Table), href "#" ] [ text "Cancel" ] , if not newOrg then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/PersonManage.elm b/modules/webapp/src/main/elm/Comp/PersonManage.elm index 14241bf0..7d73fc1e 100644 --- a/modules/webapp/src/main/elm/Comp/PersonManage.elm +++ b/modules/webapp/src/main/elm/Comp/PersonManage.elm @@ -326,14 +326,14 @@ viewForm settings model = , a [ class "ui secondary button" , onClick (SetViewMode Table) - , href "" + , href "#" ] [ text "Cancel" ] , if not newPerson then a [ class "ui right floated red button" - , href "" + , href "#" , onClick RequestDelete ] [ text "Delete" ] diff --git a/modules/webapp/src/main/elm/Comp/SourceManage.elm b/modules/webapp/src/main/elm/Comp/SourceManage.elm index 4e79cd8c..42a33446 100644 --- a/modules/webapp/src/main/elm/Comp/SourceManage.elm +++ b/modules/webapp/src/main/elm/Comp/SourceManage.elm @@ -389,11 +389,11 @@ viewForm flags settings model = , button [ class "ui primary button", type_ "submit" ] [ text "Submit" ] - , a [ class "ui secondary button", onClick SetTableView, href "" ] + , a [ class "ui secondary button", onClick SetTableView, href "#" ] [ text "Cancel" ] , if not newSource then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/TagManage.elm b/modules/webapp/src/main/elm/Comp/TagManage.elm index 41a2f708..feff8c18 100644 --- a/modules/webapp/src/main/elm/Comp/TagManage.elm +++ b/modules/webapp/src/main/elm/Comp/TagManage.elm @@ -291,11 +291,11 @@ viewForm model = , button [ class "ui primary button", type_ "submit" ] [ text "Submit" ] - , a [ class "ui secondary button", onClick (SetViewMode Table), href "" ] + , a [ class "ui secondary button", onClick (SetViewMode Table), href "#" ] [ text "Cancel" ] , if not newTag then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/UserManage.elm b/modules/webapp/src/main/elm/Comp/UserManage.elm index a8d077b3..d007e562 100644 --- a/modules/webapp/src/main/elm/Comp/UserManage.elm +++ b/modules/webapp/src/main/elm/Comp/UserManage.elm @@ -253,11 +253,11 @@ viewForm settings model = , button [ class "ui primary button", type_ "submit" ] [ text "Submit" ] - , a [ class "ui secondary button", onClick (SetViewMode Table), href "" ] + , a [ class "ui secondary button", onClick (SetViewMode Table), href "#" ] [ text "Cancel" ] , if not newUser then - a [ class "ui right floated red button", href "", onClick RequestDelete ] + a [ class "ui right floated red button", href "#", onClick RequestDelete ] [ text "Delete" ] else diff --git a/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm b/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm index 8421bdb5..a985f4d4 100644 --- a/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm +++ b/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm @@ -120,11 +120,11 @@ view2 active settings model = ] , div [ class "content" ] [ div [ class "ui buttons" ] - [ a [ class "ui primary button", onClick ConfirmDelete, href "" ] + [ a [ class "ui primary button", onClick ConfirmDelete, href "#" ] [ text settings.confirmButton ] , div [ class "or" ] [] - , a [ class "ui secondary button", onClick Disable, href "" ] + , a [ class "ui secondary button", onClick Disable, href "#" ] [ text settings.cancelButton ] ] diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index b4c93634..435aa293 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -87,7 +87,7 @@ view flags settings model = [ class "borderless item" , onClick (DoSearch BasicSearch) , title "Run search query" - , href "" + , href "#" , disabled model.searchInProgress ] [ i diff --git a/modules/webapp/src/main/elm/Page/NewInvite/View.elm b/modules/webapp/src/main/elm/Page/NewInvite/View.elm index 7e6f8b50..95ef0723 100644 --- a/modules/webapp/src/main/elm/Page/NewInvite/View.elm +++ b/modules/webapp/src/main/elm/Page/NewInvite/View.elm @@ -51,7 +51,7 @@ view flags model = ] [ text "Submit" ] - , a [ class "ui right floated button", href "", onClick Reset ] + , a [ class "ui right floated button", href "#", onClick Reset ] [ text "Reset" ] , resultMessage model diff --git a/modules/webapp/src/main/elm/Page/Upload/View.elm b/modules/webapp/src/main/elm/Page/Upload/View.elm index b5b96ae3..02fb04b6 100644 --- a/modules/webapp/src/main/elm/Page/Upload/View.elm +++ b/modules/webapp/src/main/elm/Page/Upload/View.elm @@ -25,10 +25,10 @@ view mid model = ] , Html.map DropzoneMsg (Comp.Dropzone.view model.dropzone) , div [ class "ui bottom attached segment" ] - [ a [ class "ui primary button", href "", onClick SubmitUpload ] + [ a [ class "ui primary button", href "#", onClick SubmitUpload ] [ text "Submit" ] - , a [ class "ui secondary button", href "", onClick Clear ] + , a [ class "ui secondary button", href "#", onClick Clear ] [ text "Reset" ] ] @@ -91,7 +91,7 @@ renderSuccessMsg public _ = ] , p [] [ text "Click " - , a [ class "ui link", href "", onClick Clear ] + , a [ class "ui link", href "#", onClick Clear ] [ text "Reset" ] , text " to upload more files." From 4ec133b0b98212cfaaa35787ce2a76667b61d5fc Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 23:06:58 +0100 Subject: [PATCH 37/38] Remove unused imports --- modules/webapp/src/main/elm/Comp/SearchMenu.elm | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index 3d023ed1..e60bbf99 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -17,15 +17,12 @@ module Comp.SearchMenu exposing import Api import Api.Model.Equipment exposing (Equipment) import Api.Model.EquipmentList exposing (EquipmentList) -import Api.Model.FolderItem exposing (FolderItem) -import Api.Model.FolderList exposing (FolderList) import Api.Model.FolderStats exposing (FolderStats) import Api.Model.IdName exposing (IdName) import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.PersonList exposing (PersonList) import Api.Model.ReferenceList exposing (ReferenceList) import Api.Model.SearchStats exposing (SearchStats) -import Api.Model.TagCloud exposing (TagCloud) import Comp.CustomFieldMultiInput import Comp.DatePicker import Comp.Dropdown exposing (isDropdownChangeMsg) @@ -42,7 +39,6 @@ import Html exposing (..) import Html.Attributes exposing (..) import Html.Events exposing (onCheck, onClick, onInput) import Http -import Util.Folder import Util.Html exposing (KeyCode(..)) import Util.ItemDragDrop as DD import Util.Maybe From 36858da624ea0122b382908ad81e0d7df046eb36 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 23:07:04 +0100 Subject: [PATCH 38/38] Fix search condition for empty items set --- .../store/src/main/scala/docspell/store/queries/QItem.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index b0184aaf..c3807351 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -213,7 +213,9 @@ object QItem { q.dueDateFrom.map(d => i.dueDate > d) &&? q.dueDateTo.map(d => i.dueDate < d) &&? q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? - q.itemIds.flatMap(s => Nel.fromList(s.toList)).map(nel => i.id.in(nel)) &&? + q.itemIds.map(s => + Nel.fromList(s.toList).map(nel => i.id.in(nel)).getOrElse(i.id.isNull) + ) &&? TagItemName .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) .map(subsel => i.id.in(subsel)) &&?