TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-05 10:59:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start converting QItem

Browse Source
This commit is contained in:
Eike Kettner 2020-12-13 22:56:19 +01:00
parent a355767fdb
commit 35c62049f5
10 changed files with 170 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
View File

@ -121,7 +121,7 @@ object CreateItem {
private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
Task { ctx =>
val states = ItemState.invalidStates.toList.toSet
val states = ItemState.invalidStates
val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet
for {
cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states))

2
modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
View File

@ -105,7 +105,7 @@ object ItemHandler {
private def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, Args, Unit] =
Task { ctx =>
val states = ItemState.invalidStates.toList.toSet
val states = ItemState.invalidStates
for {
items <- ctx.store.transact(
QItem.findByFileIds(ctx.args.files.map(_.fileMetaId), states)

7
modules/store/src/main/scala/docspell/store/qb/CteBind.scala
View File

@ -1,9 +1,12 @@
package docspell.store.qb
case class CteBind(name: TableDef, select: Select) {}
case class CteBind(name: TableDef, coldef: Vector[Column[_]], select: Select) {}
object CteBind {
def apply(t: (TableDef, Select)): CteBind =
CteBind(t._1, t._2)
CteBind(t._1, Vector.empty, t._2)
def apply(name: TableDef, col: Column[_], cols: Column[_]*)(select: Select): CteBind =
CteBind(name, cols.toVector.prepended(col), select)
}

7
modules/store/src/main/scala/docspell/store/qb/DSL.scala
View File

@ -25,6 +25,13 @@ trait DSL extends DoobieMeta {
def withCte(cte: (TableDef, Select), more: (TableDef, Select)*): DSL.WithCteDsl =
DSL.WithCteDsl(CteBind(cte), more.map(CteBind.apply).toVector)
def withCte(
name: TableDef,
col: Column[_],
cols: Column[_]*
): Select => DSL.WithCteDsl =
sel => DSL.WithCteDsl(CteBind(name, col, cols: _*)(sel), Vector.empty)
def select(cond: Condition): Nel[SelectExpr] =
Nel.of(SelectExpr.SelectCondition(cond, None))

48
modules/store/src/main/scala/docspell/store/qb/Select.scala
View File

@ -21,9 +21,21 @@ sealed trait Select {
def limit(n: Int): Select =
this match {
case Select.Limit(q, _) => Select.Limit(q, n)
case _ => Select.Limit(this, n)
case Select.Limit(q, _) =>
Select.Limit(q, n)
case _ =>
Select.Limit(this, n)
}
def appendCte(next: CteBind): Select =
this match {
case Select.WithCte(cte, ctes, query) =>
Select.WithCte(cte, ctes :+ next, query)
case _ =>
Select.WithCte(next, Vector.empty, this)
}
def appendSelect(e: SelectExpr): Select
}
object Select {
@ -69,16 +81,34 @@ object Select {
copy(where = c)
def where(c: Condition): SimpleSelect =
copy(where = Some(c))
def appendSelect(e: SelectExpr): SimpleSelect =
copy(projection = projection.append(e))
}
case class Union(q: Select, qs: Vector[Select]) extends Select
case class Union(q: Select, qs: Vector[Select]) extends Select {
def appendSelect(e: SelectExpr): Union =
copy(q = q.appendSelect(e))
}
case class Intersect(q: Select, qs: Vector[Select]) extends Select
case class Intersect(q: Select, qs: Vector[Select]) extends Select {
def appendSelect(e: SelectExpr): Intersect =
copy(q = q.appendSelect(e))
}
case class Ordered(q: Select, orderBy: OrderBy, orderBys: Vector[OrderBy])
extends Select
case class Limit(q: Select, limit: Int) extends Select
case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select
extends Select {
def appendSelect(e: SelectExpr): Ordered =
copy(q = q.appendSelect(e))
}
case class Limit(q: Select, limit: Int) extends Select {
def appendSelect(e: SelectExpr): Limit =
copy(q = q.appendSelect(e))
}
case class WithCte(cte: CteBind, ctes: Vector[CteBind], query: Select) extends Select {
def appendSelect(e: SelectExpr): WithCte =
copy(query = query.appendSelect(e))
}
}

2
modules/store/src/main/scala/docspell/store/qb/TableDef.scala
View File

@ -8,7 +8,7 @@ trait TableDef {
object TableDef {
def apply(table: String, aliasName: Option[String] = None): TableDef =
def apply(table: String, aliasName: Option[String] = None): BasicTable =
BasicTable(table, aliasName)
final case class BasicTable(tableName: String, alias: Option[String]) extends TableDef {

1
modules/store/src/main/scala/docspell/store/qb/impl/CommonBuilder.scala
View File

@ -18,3 +18,4 @@ trait CommonBuilder {
def appendAs(alias: Option[String]): Fragment =
alias.map(a => fr" AS" ++ Fragment.const(a)).getOrElse(Fragment.empty)
}
object CommonBuilder extends CommonBuilder

4
modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala
View File

@ -24,10 +24,10 @@ object DBFunctionBuilder extends CommonBuilder {
case DBFunction.Coalesce(expr, exprs) =>
val v = exprs.prepended(expr).map(SelectExprBuilder.build)
sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ sql")"
sql"COALESCE(" ++ v.reduce(_ ++ comma ++ _) ++ fr")"
case DBFunction.Power(expr, base) =>
sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ sql")"
sql"POWER($base, " ++ SelectExprBuilder.build(expr) ++ fr")"
case DBFunction.Calc(op, left, right) =>
SelectExprBuilder.build(left) ++

18
modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala
View File

@ -1,9 +1,9 @@
package docspell.store.qb.impl
import docspell.store.qb._
import _root_.doobie.implicits._
import _root_.doobie.{Query => _, _}
import cats.data.NonEmptyList
object SelectBuilder {
val comma = fr","
@ -74,5 +74,19 @@ object SelectBuilder {
}
def buildCte(bind: CteBind): Fragment =
Fragment.const(bind.name.tableName) ++ sql"AS (" ++ build(bind.select) ++ sql")"
bind match {
case CteBind(name, cols, select) =>
val colDef =
NonEmptyList
.fromFoldable(cols)
.map(nel =>
nel
.map(col => CommonBuilder.columnNoPrefix(col))
.reduceLeft(_ ++ comma ++ _)
)
.map(f => sql"(" ++ f ++ sql")")
.getOrElse(Fragment.empty)
Fragment.const0(name.tableName) ++ colDef ++ sql" AS (" ++ build(select) ++ sql")"
}
}

235
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -6,7 +6,6 @@ import cats.effect.Sync
import cats.effect.concurrent.Ref
import cats.implicits._
import fs2.Stream
import docspell.common.syntax.all._
import docspell.common.{IdRef, _}
import docspell.store.Store
@ -14,7 +13,6 @@ import docspell.store.impl.Implicits._
import docspell.store.impl._
import docspell.store.qb.Select
import docspell.store.records._
import bitpeace.FileMeta
import doobie._
import doobie.implicits._
@ -583,19 +581,18 @@ object QItem {
}
private def findAttachmentLight(item: Ident): ConnectionIO[List[AttachmentLight]] = {
val aId = RAttachment.Columns.id.prefix("a")
val aItem = RAttachment.Columns.itemId.prefix("a")
val aPos = RAttachment.Columns.position.prefix("a")
val aName = RAttachment.Columns.name.prefix("a")
val mId = RAttachmentMeta.Columns.id.prefix("m")
val mPages = RAttachmentMeta.Columns.pages.prefix("m")
import docspell.store.qb._
import docspell.store.qb.DSL._
val cols = Seq(aId, aPos, aName, mPages)
val join = RAttachment.table ++
fr"a LEFT OUTER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId)
val cond = aItem.is(item)
val a = RAttachment.as("a")
val m = RAttachmentMeta.as("m")
selectSimple(cols, join, cond).query[AttachmentLight].to[List]
Select(
select(a.id, a.position, a.name, m.pages),
from(a)
.leftJoin(m, m.id === a.id),
a.itemId === item
).build.query[AttachmentLight].to[List]
}
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
@ -609,108 +606,73 @@ object QItem {
private def findByFileIdsQuery(
fileMetaIds: NonEmptyList[Ident],
limit: Option[Int],
states: Set[ItemState]
): Fragment = {
val IC = RItem.Columns.all.map(_.prefix("i"))
val aItem = RAttachment.Columns.itemId.prefix("a")
val aId = RAttachment.Columns.id.prefix("a")
val aFileId = RAttachment.Columns.fileId.prefix("a")
val iId = RItem.Columns.id.prefix("i")
val iState = RItem.Columns.state.prefix("i")
val sId = RAttachmentSource.Columns.id.prefix("s")
val sFileId = RAttachmentSource.Columns.fileId.prefix("s")
val rId = RAttachmentArchive.Columns.id.prefix("r")
val rFileId = RAttachmentArchive.Columns.fileId.prefix("r")
val m1 = RFileMeta.as("m1")
val m2 = RFileMeta.as("m2")
val m3 = RFileMeta.as("m3")
val m1Id = m1.id.column
val m2Id = m2.id.column
val m3Id = m3.id.column
val filemetaTable = Fragment.const(RFileMeta.T.tableName)
states: Option[NonEmptyList[ItemState]]
): Select.SimpleSelect = {
import docspell.store.qb._
import docspell.store.qb.DSL._
val from =
RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++
fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++
fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++
fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++
fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++
fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId)
val i = RItem.as("i")
val a = RAttachment.as("a")
val s = RAttachmentSource.as("s")
val r = RAttachmentArchive.as("r")
val fileCond =
or(m1Id.isIn(fileMetaIds), m2Id.isIn(fileMetaIds), m3Id.isIn(fileMetaIds))
val cond = NonEmptyList.fromList(states.toList) match {
case Some(nel) =>
and(fileCond, iState.isIn(nel))
case None =>
fileCond
}
val q = selectSimple(IC, from, cond)
limit match {
case Some(n) => q ++ fr"LIMIT $n"
case None => q
}
Select(
select(i.all),
from(i)
.innerJoin(a, a.itemId === i.id)
.innerJoin(s, s.id === a.id)
.leftJoin(r, r.id === a.id),
(a.fileId.in(fileMetaIds) ||
s.fileId.in(fileMetaIds) ||
r.fileId.in(fileMetaIds)) &&? states.map(nel => i.state.in(nel))
)
}
def findOneByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Option[RItem]] =
NonEmptyList.fromList(fileMetaIds.toList) match {
case Some(nel) =>
findByFileIdsQuery(nel, Some(1), Set.empty).query[RItem].option
findByFileIdsQuery(nel, None).limit(1).build.query[RItem].option
case None =>
(None: Option[RItem]).pure[ConnectionIO]
}
def findByFileIds(
fileMetaIds: Seq[Ident],
states: Set[ItemState]
states: NonEmptyList[ItemState]
): ConnectionIO[Vector[RItem]] =
NonEmptyList.fromList(fileMetaIds.toList) match {
case Some(nel) =>
findByFileIdsQuery(nel, None, states).query[RItem].to[Vector]
findByFileIdsQuery(nel, states.some).build.query[RItem].to[Vector]
case None =>
Vector.empty[RItem].pure[ConnectionIO]
}
def findByChecksum(checksum: String, collective: Ident): ConnectionIO[Vector[RItem]] = {
val IC = RItem.Columns.all.map(_.prefix("i"))
val aItem = RAttachment.Columns.itemId.prefix("a")
val aId = RAttachment.Columns.id.prefix("a")
val aFileId = RAttachment.Columns.fileId.prefix("a")
val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i")
val sId = RAttachmentSource.Columns.id.prefix("s")
val sFileId = RAttachmentSource.Columns.fileId.prefix("s")
val rId = RAttachmentArchive.Columns.id.prefix("r")
val rFileId = RAttachmentArchive.Columns.fileId.prefix("r")
import docspell.store.qb._
import docspell.store.qb.DSL._
val m1 = RFileMeta.as("m1")
val m2 = RFileMeta.as("m2")
val m3 = RFileMeta.as("m3")
val m1Id = m1.id.column
val m2Id = m2.id.column
val m3Id = m3.id.column
val m1Checksum = m1.checksum.column
val m2Checksum = m2.checksum.column
val m3Checksum = m3.checksum.column
val filemetaTable = Fragment.const(RFileMeta.T.tableName)
val from =
RItem.table ++ fr"i INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++
fr"INNER JOIN" ++ RAttachmentSource.table ++ fr"s ON" ++ aId.is(sId) ++
fr"INNER JOIN" ++ filemetaTable ++ fr"m1 ON" ++ m1Id.is(aFileId) ++
fr"INNER JOIN" ++ filemetaTable ++ fr"m2 ON" ++ m2Id.is(sFileId) ++
fr"LEFT OUTER JOIN" ++ RAttachmentArchive.table ++ fr"r ON" ++ aId.is(rId) ++
fr"LEFT OUTER JOIN" ++ filemetaTable ++ fr"m3 ON" ++ m3Id.is(rFileId)
val i = RItem.as("i")
val a = RAttachment.as("a")
val s = RAttachmentSource.as("s")
val r = RAttachmentArchive.as("r")
selectSimple(
IC,
from,
and(
or(m1Checksum.is(checksum), m2Checksum.is(checksum), m3Checksum.is(checksum)),
iColl.is(collective)
Select(
select(i.all),
from(i)
.innerJoin(a, a.itemId === i.id)
.innerJoin(s, s.id === a.id)
.innerJoin(m1, m1.id === a.fileId)
.innerJoin(m2, m2.id === s.fileId)
.leftJoin(r, r.id === a.id)
.leftJoin(m3, m3.id === r.fileId),
where(
i.cid === collective &&
(m1.checksum === checksum || m2.checksum === checksum || m3.checksum === checksum)
)
).query[RItem]
.to[Vector]
).build.query[RItem].to[Vector]
}
final case class NameAndNotes(
@ -724,15 +686,16 @@ object QItem {
coll: Option[Ident],
chunkSize: Int
): Stream[ConnectionIO, NameAndNotes] = {
val iId = RItem.Columns.id
val iColl = RItem.Columns.cid
val iName = RItem.Columns.name
val iFolder = RItem.Columns.folder
val iNotes = RItem.Columns.notes
import docspell.store.qb._
import docspell.store.qb.DSL._
val cols = Seq(iId, iColl, iFolder, iName, iNotes)
val where = coll.map(cid => iColl.is(cid)).getOrElse(Fragment.empty)
selectSimple(cols, RItem.table, where)
val i = RItem.as("i")
Select(
select(i.id, i.cid, i.folder, i.name, i.notes),
from(i)
).where(coll.map(cid => i.cid === cid))
.build
.query[NameAndNotes]
.streamWithChunkSize(chunkSize)
}
@ -741,15 +704,13 @@ object QItem {
collective: Ident,
chunkSize: Int
): Stream[ConnectionIO, Ident] = {
val cols = Seq(RItem.Columns.id)
val iColl = RItem.Columns.cid
val iState = RItem.Columns.state
(selectSimple(
cols,
RItem.table,
and(iColl.is(collective), iState.is(ItemState.confirmed))
) ++
orderBy(RItem.Columns.created.desc))
import docspell.store.qb._
import docspell.store.qb.DSL._
val i = RItem.as("i")
Select(i.id.s, from(i), i.cid === collective && i.state === ItemState.confirmed)
.orderBy(i.created.desc)
.build
.query[Ident]
.streamWithChunkSize(chunkSize)
}
@ -763,45 +724,39 @@ object QItem {
tagCategory: String,
pageSep: String
): ConnectionIO[TextAndTag] = {
val aId = RAttachment.Columns.id.prefix("a")
val aItem = RAttachment.Columns.itemId.prefix("a")
val mId = RAttachmentMeta.Columns.id.prefix("m")
val mText = RAttachmentMeta.Columns.content.prefix("m")
val tagItem = RTagItem.as("ti") //Columns.itemId.prefix("ti")
//val tiTag = RTagItem.Columns.tagId.prefix("ti")
import docspell.store.qb._
import docspell.store.qb.DSL._
val tag = RTag.as("t")
// val tId = RTag.Columns.tid.prefix("t")
// val tName = RTag.Columns.name.prefix("t")
// val tCat = RTag.Columns.category.prefix("t")
val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i")
val a = RAttachment.as("a")
val am = RAttachmentMeta.as("m")
val ti = RTagItem.as("ti")
val i = RItem.as("i")
val cte = withCTE(
"tags" -> selectSimple(
Seq(tagItem.itemId.column, tag.tid.column, tag.name.column),
Fragment.const(RTagItem.t.tableName) ++ fr"ti INNER JOIN" ++
Fragment.const(tag.tableName) ++ fr"t ON" ++ tag.tid.column
.is(tagItem.tagId.column),
and(tagItem.itemId.column.is(itemId), tag.category.column.is(tagCategory))
val tags = TableDef("tags").as("tt")
val tagsItem = Column[Ident]("itemid", tags)
val tagsTid = Column[Ident]("tid", tags)
val tagsName = Column[String]("tname", tags)
val q =
withCte(
tags -> Select(
select(ti.itemId.as(tagsItem), tag.tid.as(tagsTid), tag.name.as(tagsName)),
from(ti)
.innerJoin(tag, tag.tid === ti.tagId),
ti.itemId === itemId && tag.category === tagCategory
)
)(
Select(
select(am.content, tagsTid, tagsName),
from(i)
.innerJoin(a, a.itemId === i.id)
.innerJoin(am, a.id === am.id)
.leftJoin(tags, tagsItem === i.id),
i.id === itemId && i.cid === collective && am.content.isNotNull && am.content <> ""
)
).build
val cols = Seq(mText, tag.tid.column, tag.name.column)
val from = RItem.table ++ fr"i INNER JOIN" ++
RAttachment.table ++ fr"a ON" ++ aItem.is(iId) ++ fr"INNER JOIN" ++
RAttachmentMeta.table ++ fr"m ON" ++ aId.is(mId) ++ fr"LEFT JOIN" ++
fr"tags t ON" ++ RTagItem.t.itemId.oldColumn.prefix("t").is(iId)
val where =
and(
iId.is(itemId),
iColl.is(collective),
mText.isNotNull,
mText.isNot("")
)
val q = cte ++ selectDistinct(cols, from, where)
for {
_ <- logger.ftrace[ConnectionIO](
s"query: $q (${itemId.id}, ${collective.id}, ${tagCategory})"