TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-04 18:39:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor search to separate between a base query and user query

Browse Source 
The `findBase` is adding only strictly required conditions. Everything
else comes from the user.
This commit is contained in:
Eike Kettner 2021-02-24 21:57:41 +01:00
parent c3cdec416c
commit 186014a1c6
7 changed files with 129 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@ -164,7 +164,7 @@ object OFulltext {
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
q = Query.empty(account).copy(itemIds = itemIds.some)
q = Query.empty(account).withCond(_.copy(itemIds = itemIds.some))
res <- store.transact(QItem.searchStats(q))
} yield res
}
@ -208,7 +208,7 @@ object OFulltext {
search <- itemSearch.findItems(0)(q, Batch.all)
fq = FtsQuery(
ftsQ.query,
q.account.collective,
q.fix.account.collective,
search.map(_.id).toSet,
Set.empty,
500,
@ -220,7 +220,7 @@ object OFulltext {
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
qnext = q.copy(itemIds = items.some)
qnext = q.withCond(_.copy(itemIds = items.some))
res <- store.transact(QItem.searchStats(qnext))
} yield res
@ -253,7 +253,7 @@ object OFulltext {
val sqlResult = search(q, batch)
val fq = FtsQuery(
ftsQ.query,
q.account.collective,
q.fix.account.collective,
Set.empty,
Set.empty,
0,

4
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@ -138,7 +138,9 @@ object OItemSearch {
val search = QItem.findItems(q, maxNoteLen: Int, batch)
store
.transact(
QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)
QItem
.findItemsWithTags(q.fix.account.collective, search)
.take(batch.limit.toLong)
)
.compile
.toVector

17
modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
View File

@ -72,13 +72,16 @@ object NotifyDueItemsTask {
q =
Query
.empty(ctx.args.account)
.copy(
states = ItemState.validStates.toList,
tagsInclude = ctx.args.tagsInclude,
tagsExclude = ctx.args.tagsExclude,
dueDateFrom = ctx.args.daysBack.map(back => now - Duration.days(back.toLong)),
dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong)),
orderAsc = Some(_.dueDate)
.withOrder(orderAsc = _.dueDate)
.withCond(
_.copy(
states = ItemState.validStates.toList,
tagsInclude = ctx.args.tagsInclude,
tagsExclude = ctx.args.tagsExclude,
dueDateFrom =
ctx.args.daysBack.map(back => now - Duration.days(back.toLong)),
dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong))
)
)
res <-
ctx.store

53
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@ -145,31 +145,32 @@ trait Conversions {
def mkQuery(m: ItemSearch, account: AccountId): OItemSearch.Query =
OItemSearch.Query(
account,
m.name,
if (m.inbox) Seq(ItemState.Created)
else ItemState.validStates.toList,
m.direction,
m.corrPerson,
m.corrOrg,
m.concPerson,
m.concEquip,
m.folder,
m.tagsInclude.map(Ident.unsafe),
m.tagsExclude.map(Ident.unsafe),
m.tagCategoriesInclude,
m.tagCategoriesExclude,
m.dateFrom,
m.dateUntil,
m.dueDateFrom,
m.dueDateUntil,
m.allNames,
m.itemSubset
.map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet)
.filter(_.nonEmpty),
m.customValues.map(mkCustomValue),
m.source,
None
OItemSearch.Query.Fix(account, None),
OItemSearch.Query.QueryCond(
m.name,
if (m.inbox) Seq(ItemState.Created)
else ItemState.validStates.toList,
m.direction,
m.corrPerson,
m.corrOrg,
m.concPerson,
m.concEquip,
m.folder,
m.tagsInclude.map(Ident.unsafe),
m.tagsExclude.map(Ident.unsafe),
m.tagCategoriesInclude,
m.tagCategoriesExclude,
m.dateFrom,
m.dateUntil,
m.dueDateFrom,
m.dueDateUntil,
m.allNames,
m.itemSubset
.map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet)
.filter(_.nonEmpty),
m.customValues.map(mkCustomValue),
m.source
)
)
def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue =
@ -182,7 +183,7 @@ trait Conversions {
ItemLightGroup(g._1, g._2.map(mkItemLight).toList)
val gs =
groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
}

48
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -117,7 +117,7 @@ object QItem {
.map(nel => intersect(nel.map(singleSelect)))
}
private def findItemsBase(q: Query, noteMaxLen: Int): Select = {
private def findItemsBase(q: Query.Fix, noteMaxLen: Int): Select = {
object Attachs extends TableDef {
val tableName = "attachs"
val aliasName = "cta"
@ -128,7 +128,7 @@ object QItem {
val coll = q.account.collective
val baseSelect = Select(
Select(
select(
i.id.s,
i.name.s,
@ -172,27 +172,23 @@ object QItem {
.leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll)
.leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll),
where(
i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&
or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account)))
i.cid === coll && or(
i.folder.isNull,
i.folder.in(QFolder.findMemberFolderIds(q.account))
)
)
).distinct.orderBy(
q.orderAsc
.map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
.getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
)
findCustomFieldValuesForColl(coll, q.customValues) match {
case Some(itemIds) =>
baseSelect.changeWhere(c => c && i.id.in(itemIds))
case None =>
baseSelect
}
}
def queryCondition(q: Query): Condition =
def queryCondition(coll: Ident, q: Query.QueryCond): Condition =
Condition.unit &&?
q.direction.map(d => i.incoming === d) &&?
q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&?
Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&?
q.allNames
.map(QueryWildcard.lower)
.map(n =>
@ -221,15 +217,17 @@ object QItem {
.map(subsel => i.id.in(subsel)) &&?
TagItemName
.itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl)
.map(subsel => i.id.notIn(subsel))
.map(subsel => i.id.notIn(subsel)) &&?
findCustomFieldValuesForColl(coll, q.customValues)
.map(itemIds => i.id.in(itemIds))
def findItems(
q: Query,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] = {
val sql = findItemsBase(q, maxNoteLen)
.changeWhere(c => c && queryCondition(q))
val sql = findItemsBase(q.fix, maxNoteLen)
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.limit(batch)
.build
logger.trace(s"List $batch items: $sql")
@ -251,10 +249,10 @@ object QItem {
.innerJoin(i, i.id === ti.itemId)
val tagCloud =
findItemsBase(q, 0).unwrap
findItemsBase(q.fix, 0).unwrap
.withSelect(select(tag.all).append(count(i.id).as("num")))
.changeFrom(_.prepend(tagFrom))
.changeWhere(c => c && queryCondition(q))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.groupBy(tag.tid)
.build
.query[TagCount]
@ -264,24 +262,24 @@ object QItem {
// are not included they are fetched separately
for {
existing <- tagCloud
other <- RTag.findOthers(q.account.collective, existing.map(_.tag.tagId))
other <- RTag.findOthers(q.fix.account.collective, existing.map(_.tag.tagId))
} yield existing ++ other.map(TagCount(_, 0))
}
def searchCountSummary(q: Query): ConnectionIO[Int] =
findItemsBase(q, 0).unwrap
findItemsBase(q.fix, 0).unwrap
.withSelect(Nel.of(count(i.id).as("num")))
.changeWhere(c => c && queryCondition(q))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.build
.query[Int]
.unique
def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = {
val fu = RUser.as("fu")
findItemsBase(q, 0).unwrap
findItemsBase(q.fix, 0).unwrap
.withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num")))
.changeFrom(_.innerJoin(fu, fu.uid === f.owner))
.changeWhere(c => c && queryCondition(q))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.groupBy(f.id, f.name, f.owner, fu.login)
.build
.query[FolderCount]
@ -295,9 +293,9 @@ object QItem {
.innerJoin(i, i.id === cv.itemId)
val base =
findItemsBase(q, 0).unwrap
findItemsBase(q.fix, 0).unwrap
.changeFrom(_.prepend(fieldJoin))
.changeWhere(c => c && queryCondition(q))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.groupBy(GroupBy(cf.all))
val basicFields = Nel.of(
@ -374,7 +372,7 @@ object QItem {
)
)
val from = findItemsBase(q, maxNoteLen)
val from = findItemsBase(q.fix, maxNoteLen)
.appendCte(cte)
.appendSelect(Tids.weight.s)
.changeFrom(_.innerJoin(Tids, Tids.itemId === i.id))

109
modules/store/src/main/scala/docspell/store/queries/Query.scala
View File

@ -1,57 +1,70 @@
package docspell.store.queries
import docspell.common._
import docspell.store.qb.Column
import docspell.store.records.RItem
case class Query(
account: AccountId,
name: Option[String],
states: Seq[ItemState],
direction: Option[Direction],
corrPerson: Option[Ident],
corrOrg: Option[Ident],
concPerson: Option[Ident],
concEquip: Option[Ident],
folder: Option[Ident],
tagsInclude: List[Ident],
tagsExclude: List[Ident],
tagCategoryIncl: List[String],
tagCategoryExcl: List[String],
dateFrom: Option[Timestamp],
dateTo: Option[Timestamp],
dueDateFrom: Option[Timestamp],
dueDateTo: Option[Timestamp],
allNames: Option[String],
itemIds: Option[Set[Ident]],
customValues: Seq[CustomValue],
source: Option[String],
orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]]
)
case class Query(fix: Query.Fix, cond: Query.QueryCond) {
def withCond(f: Query.QueryCond => Query.QueryCond): Query =
copy(cond = f(cond))
def withOrder(orderAsc: RItem.Table => Column[_]): Query =
copy(fix = fix.copy(orderAsc = Some(orderAsc)))
}
object Query {
case class Fix(account: AccountId, orderAsc: Option[RItem.Table => Column[_]])
case class QueryCond(
name: Option[String],
states: Seq[ItemState],
direction: Option[Direction],
corrPerson: Option[Ident],
corrOrg: Option[Ident],
concPerson: Option[Ident],
concEquip: Option[Ident],
folder: Option[Ident],
tagsInclude: List[Ident],
tagsExclude: List[Ident],
tagCategoryIncl: List[String],
tagCategoryExcl: List[String],
dateFrom: Option[Timestamp],
dateTo: Option[Timestamp],
dueDateFrom: Option[Timestamp],
dueDateTo: Option[Timestamp],
allNames: Option[String],
itemIds: Option[Set[Ident]],
customValues: Seq[CustomValue],
source: Option[String]
)
object QueryCond {
val empty =
QueryCond(
None,
Seq.empty,
None,
None,
None,
None,
None,
None,
Nil,
Nil,
Nil,
Nil,
None,
None,
None,
None,
None,
None,
Seq.empty,
None
)
}
def empty(account: AccountId): Query =
Query(
account,
None,
Seq.empty,
None,
None,
None,
None,
None,
None,
Nil,
Nil,
Nil,
Nil,
None,
None,
None,
None,
None,
None,
Seq.empty,
None,
None
)
Query(Fix(account, None), QueryCond.empty)
}

2
modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala
View File

@ -23,7 +23,7 @@ object ItemQueryGeneratorTest extends SimpleTestSuite {
RAttachmentMeta.as("m")
)
test("migration") {
test("basic test") {
val q = ItemQueryParser
.parseUnsafe("(& name:hello date>=2020-02-01 (| source=expense folder=test ))")
val cond = ItemQueryGenerator(tables, Ident.unsafe("coll"))(q)