TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-04 10:29:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add more convenient date parsers and some basic macros

Browse Source
This commit is contained in:
Eike Kettner 2021-02-28 16:11:25 +01:00
parent af73b59ec2
commit 9013d9264e
23 changed files with 445 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.sbt
View File

@ -275,7 +275,9 @@ val query =
.settings(
name := "docspell-query",
libraryDependencies +=
Dependencies.catsParseJS.value
Dependencies.catsParseJS.value,
libraryDependencies +=
Dependencies.scalaJavaTime.value
)
.jsSettings(
Test / fork := false

6
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@ -159,13 +159,14 @@ object OFulltext {
for {
folder <- store.transact(QFolder.getMemberFolders(account))
now <- Timestamp.current[F]
itemIds <- fts
.searchAll(fq.withFolders(folder))
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
q = Query.empty(account).withFix(_.copy(itemIds = itemIds.some))
res <- store.transact(QItem.searchStats(q))
res <- store.transact(QItem.searchStats(now.toUtcDate)(q))
} yield res
}
@ -221,7 +222,8 @@ object OFulltext {
.compile
.to(Set)
qnext = q.withFix(_.copy(itemIds = items.some))
res <- store.transact(QItem.searchStats(qnext))
now <- Timestamp.current[F]
res <- store.transact(QItem.searchStats(now.toUtcDate)(qnext))
} yield res
// Helper

44
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@ -127,27 +127,39 @@ object OItemSearch {
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
.compile
.toVector
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap { today =>
store
.transact(
QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong)
)
.compile
.toVector
}
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, maxNoteLen: Int, batch)
store
.transact(
QItem
.findItemsWithTags(q.fix.account.collective, search)
.take(batch.limit.toLong)
)
.compile
.toVector
}
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
for {
now <- Timestamp.current[F]
search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch)
res <- store
.transact(
QItem
.findItemsWithTags(q.fix.account.collective, search)
.take(batch.limit.toLong)
)
.compile
.toVector
} yield res
def findItemsSummary(q: Query): F[SearchSummary] =
store.transact(QItem.searchStats(q))
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap(today => store.transact(QItem.searchStats(today)(q)))
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store

6
modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
View File

@ -85,7 +85,11 @@ object NotifyDueItemsTask {
)
res <-
ctx.store
.transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
.transact(
QItem
.findItems(q, now.toUtcDate, 0, Batch.limit(maxItems))
.take(maxItems.toLong)
)
.compile
.toVector
} yield res

32
modules/query/src/main/scala/docspell/query/Date.scala
View File

@ -1,14 +1,32 @@
package docspell.query
sealed trait Date
object Date {
def apply(y: Int, m: Int, d: Int): Date =
Local(y, m, d)
import java.time.LocalDate
import java.time.Period
def apply(ms: Long): Date =
import cats.implicits._
sealed trait Date
object Date {
def apply(y: Int, m: Int, d: Int): Either[Throwable, DateLiteral] =
Either.catchNonFatal(Local(LocalDate.of(y, m, d)))
def apply(ms: Long): DateLiteral =
Millis(ms)
final case class Local(year: Int, month: Int, day: Int) extends Date
sealed trait DateLiteral extends Date
final case class Millis(ms: Long) extends Date
final case class Local(date: LocalDate) extends DateLiteral
final case class Millis(ms: Long) extends DateLiteral
case object Today extends DateLiteral
sealed trait CalcDirection
object CalcDirection {
case object Plus extends CalcDirection
case object Minus extends CalcDirection
}
case class Calc(date: DateLiteral, calc: CalcDirection, period: Period) extends Date
}

23
modules/query/src/main/scala/docspell/query/ItemQuery.scala
View File

@ -40,6 +40,7 @@ object ItemQuery {
case object ItemName extends StringAttr
case object ItemSource extends StringAttr
case object ItemNotes extends StringAttr
case object ItemId extends StringAttr
case object Date extends DateAttr
case object DueDate extends DateAttr
@ -69,6 +70,11 @@ object ItemQuery {
final case class StringProperty(attr: StringAttr, value: String) extends Property
final case class DateProperty(attr: DateAttr, value: Date) extends Property
def apply(sa: StringAttr, value: String): Property =
StringProperty(sa, value)
def apply(da: DateAttr, value: Date): Property =
DateProperty(da, value)
}
sealed trait Expr {
@ -88,6 +94,8 @@ object ItemQuery {
final case class Exists(field: Attr) extends Expr
final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr
final case class InboxExpr(inbox: Boolean) extends Expr
final case class DirectionExpr(incoming: Boolean) extends Expr
final case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr
final case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr
@ -97,6 +105,21 @@ object ItemQuery {
extends Expr
final case class Fulltext(query: String) extends Expr
def or(expr0: Expr, exprs: Expr*): OrExpr =
OrExpr(Nel.of(expr0, exprs: _*))
def and(expr0: Expr, exprs: Expr*): AndExpr =
AndExpr(Nel.of(expr0, exprs: _*))
def string(op: Operator, attr: StringAttr, value: String): SimpleExpr =
SimpleExpr(op, Property(attr, value))
def like(attr: StringAttr, value: String): SimpleExpr =
string(Operator.Like, attr, value)
def date(op: Operator, attr: DateAttr, value: Date): SimpleExpr =
SimpleExpr(op, Property(attr, value))
}
}

2
modules/query/src/main/scala/docspell/query/ItemQueryParser.scala
View File

@ -16,7 +16,7 @@ object ItemQueryParser {
ExprParser
.parseQuery(in)
.left
.map(pe => s"Error parsing: '$input': $pe")
.map(pe => s"Error parsing: '$input': $pe") //TODO
.map(q => q.copy(expr = ExprUtil.reduce(q.expr)))
}

34
modules/query/src/main/scala/docspell/query/internal/AttrParser.scala
View File

@ -7,57 +7,60 @@ import docspell.query.ItemQuery.Attr
object AttrParser {
val name: P[Attr.StringAttr] =
P.ignoreCase("name").map(_ => Attr.ItemName)
P.ignoreCase("name").as(Attr.ItemName)
val source: P[Attr.StringAttr] =
P.ignoreCase("source").map(_ => Attr.ItemSource)
P.ignoreCase("source").as(Attr.ItemSource)
val id: P[Attr.StringAttr] =
P.ignoreCase("id").map(_ => Attr.ItemId)
P.ignoreCase("id").as(Attr.ItemId)
val date: P[Attr.DateAttr] =
P.ignoreCase("date").map(_ => Attr.Date)
P.ignoreCase("date").as(Attr.Date)
val notes: P[Attr.StringAttr] =
P.ignoreCase("notes").as(Attr.ItemNotes)
val dueDate: P[Attr.DateAttr] =
P.stringIn(List("dueDate", "due", "due-date")).map(_ => Attr.DueDate)
P.stringIn(List("dueDate", "due", "due-date")).as(Attr.DueDate)
val corrOrgId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.id", "corr.org.id"))
.map(_ => Attr.Correspondent.OrgId)
.as(Attr.Correspondent.OrgId)
val corrOrgName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.name", "corr.org.name"))
.map(_ => Attr.Correspondent.OrgName)
.as(Attr.Correspondent.OrgName)
val corrPersId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.id", "corr.pers.id"))
.map(_ => Attr.Correspondent.PersonId)
.as(Attr.Correspondent.PersonId)
val corrPersName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.name", "corr.pers.name"))
.map(_ => Attr.Correspondent.PersonName)
.as(Attr.Correspondent.PersonName)
val concPersId: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.id", "conc.pers.id"))
.map(_ => Attr.Concerning.PersonId)
.as(Attr.Concerning.PersonId)
val concPersName: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.name", "conc.pers.name"))
.map(_ => Attr.Concerning.PersonName)
.as(Attr.Concerning.PersonName)
val concEquipId: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.id", "conc.equip.id"))
.map(_ => Attr.Concerning.EquipId)
.as(Attr.Concerning.EquipId)
val concEquipName: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.name", "conc.equip.name"))
.map(_ => Attr.Concerning.EquipName)
.as(Attr.Concerning.EquipName)
val folderId: P[Attr.StringAttr] =
P.ignoreCase("folder.id").map(_ => Attr.Folder.FolderId)
P.ignoreCase("folder.id").as(Attr.Folder.FolderId)
val folderName: P[Attr.StringAttr] =
P.ignoreCase("folder").map(_ => Attr.Folder.FolderName)
P.ignoreCase("folder").as(Attr.Folder.FolderName)
val dateAttr: P[Attr.DateAttr] =
P.oneOf(List(date, dueDate))
@ -68,6 +71,7 @@ object AttrParser {
name,
source,
id,
notes,
corrOrgId,
corrOrgName,
corrPersId,

6
modules/query/src/main/scala/docspell/query/internal/BasicParser.scala
View File

@ -38,4 +38,10 @@ object BasicParser {
val stringOrMore: P[Nel[String]] =
singleString.repSep(stringListSep)
val bool: P[Boolean] = {
val trueP = P.stringIn(List("yes", "true", "Yes", "True")).as(true)
val falseP = P.stringIn(List("no", "false", "No", "False")).as(false)
trueP | falseP
}
}

85
modules/query/src/main/scala/docspell/query/internal/DateParser.scala
View File

@ -1,7 +1,8 @@
package docspell.query.internal
import cats.data.NonEmptyList
import cats.implicits._
import java.time.Period
import cats.data.{NonEmptyList => Nel}
import cats.parse.{Numbers, Parser => P}
import docspell.query.Date
@ -26,21 +27,79 @@ object DateParser {
digits2.filter(n => n >= 1 && n <= 31)
private val dateSep: P[Unit] =
P.anyChar.void
P.charIn('-', '/').void
val localDateFromString: P[Date] =
((digits4 <* dateSep) ~ (month <* dateSep) ~ day).mapFilter {
case ((year, month), day) =>
Either.catchNonFatal(Date(year, month, day)).toOption
private val dateString: P[((Int, Option[Int]), Option[Int])] =
digits4 ~ (dateSep *> month).? ~ (dateSep *> day).?
private[internal] val dateFromString: P[Date.DateLiteral] =
dateString.mapFilter { case ((year, month), day) =>
Date(year, month.getOrElse(1), day.getOrElse(1)).toOption
}
val dateFromMillis: P[Date] =
longParser.map(Date.apply)
private[internal] val dateFromMillis: P[Date.DateLiteral] =
P.string("ms") *> longParser.map(Date.apply)
val localDate: P[Date] =
localDateFromString.backtrack.orElse(dateFromMillis)
private val dateFromToday: P[Date.DateLiteral] =
P.string("today").as(Date.Today)
val localDateOrMore: P[NonEmptyList[Date]] =
localDate.repSep(BasicParser.stringListSep)
val dateLiteral: P[Date.DateLiteral] =
P.oneOf(List(dateFromString, dateFromToday, dateFromMillis))
// val dateLiteralOrMore: P[NonEmptyList[Date.DateLiteral]] =
// dateLiteral.repSep(BasicParser.stringListSep)
val dateCalcDirection: P[Date.CalcDirection] =
P.oneOf(
List(
P.char('+').as(Date.CalcDirection.Plus),
P.char('-').as(Date.CalcDirection.Minus)
)
)
def periodPart(unitSuffix: Char, f: Int => Period): P[Period] =
((Numbers.nonZeroDigit ~ Numbers.digits0).void.string.soft <* P.ignoreCaseChar(
unitSuffix
))
.map(n => f(n.toInt))
private[this] val periodMonths: P[Period] =
periodPart('m', n => Period.ofMonths(n))
private[this] val periodDays: P[Period] =
periodPart('d', n => Period.ofDays(n))
val period: P[Period] =
periodDays.eitherOr(periodMonths).map(_.fold(identity, identity))
val periods: P[Period] =
period.rep.map(_.reduceLeft((p0, p1) => p0.plus(p1)))
val dateRange: P[(Date, Date)] =
((dateLiteral <* P.char(';')) ~ dateCalcDirection.eitherOr(P.char('/')) ~ period)
.map { case ((date, calc), period) =>
calc match {
case Right(Date.CalcDirection.Plus) =>
(date, Date.Calc(date, Date.CalcDirection.Plus, period))
case Right(Date.CalcDirection.Minus) =>
(Date.Calc(date, Date.CalcDirection.Minus, period), date)
case Left(_) =>
(
Date.Calc(date, Date.CalcDirection.Minus, period),
Date.Calc(date, Date.CalcDirection.Plus, period)
)
}
}
val date: P[Date] =
(dateLiteral ~ (P.char(';') *> dateCalcDirection ~ period).?).map {
case (date, Some((c, p))) =>
Date.Calc(date, c, p)
case (date, None) =>
date
}
val dateOrMore: P[Nel[Date]] =
date.repSep(BasicParser.stringListSep)
}

9
modules/query/src/main/scala/docspell/query/internal/ExprParser.scala
View File

@ -24,10 +24,11 @@ object ExprParser {
val exprParser: P[Expr] =
P.recursive[Expr] { recurse =>
val andP = and(recurse)
val orP = or(recurse)
val notP = not(recurse)
P.oneOf(SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil)
val andP = and(recurse)
val orP = or(recurse)
val notP = not(recurse)
val macros = MacroParser.all
P.oneOf(SimpleExprParser.simpleExpr :: macros :: andP :: orP :: notP :: Nil)
}
def parseQuery(input: String): Either[P.Error, ItemQuery] = {

10
modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala
View File

@ -22,10 +22,20 @@ object ExprUtil {
inner match {
case NotExpr(inner2) =>
reduce(inner2)
case InboxExpr(flag) =>
InboxExpr(!flag)
case DirectionExpr(flag) =>
DirectionExpr(!flag)
case _ =>
expr
}
case DirectionExpr(_) =>
expr
case InboxExpr(_) =>
expr
case InExpr(_, _) =>
expr

65
modules/query/src/main/scala/docspell/query/internal/MacroParser.scala Normal file
View File

@ -0,0 +1,65 @@
package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery._
object MacroParser {
private[this] val macroDef: P[String] =
P.char('$') *> BasicParser.identParser <* P.char(':')
def parser[A](macros: Map[String, P[A]]): P[A] = {
val p: P[P[A]] = macroDef.map { name =>
macros
.get(name)
.getOrElse(P.failWith(s"Unknown macro: $name"))
}
val px = (p ~ P.index ~ BasicParser.singleString).map { case ((pexpr, index), str) =>
pexpr
.parseAll(str)
.left
.map(err => err.copy(failedAtOffset = err.failedAtOffset + index))
}
P.select(px)(P.Fail)
}
// --- definitions of available macros
/** Expands in an OR expression that matches name fields of item and
* correspondent/concerning metadata.
*/
val names: P[Expr] =
P.string(P.anyChar.rep.void).map { input =>
Expr.or(
Expr.like(Attr.ItemName, input),
Expr.like(Attr.ItemNotes, input),
Expr.like(Attr.Correspondent.OrgName, input),
Expr.like(Attr.Correspondent.PersonName, input),
Expr.like(Attr.Concerning.PersonName, input),
Expr.like(Attr.Concerning.EquipName, input)
)
}
def dateRange(attr: Attr.DateAttr): P[Expr] =
DateParser.dateRange.map { case (left, right) =>
Expr.and(
Expr.date(Operator.Gte, attr, left),
Expr.date(Operator.Lte, attr, right)
)
}
// --- all macro parser
val allMacros: Map[String, P[Expr]] =
Map(
"names" -> names,
"datein" -> dateRange(Attr.Date),
"duein" -> dateRange(Attr.DueDate)
)
val all: P[Expr] =
parser(allMacros)
}

18
modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala
View File

@ -6,34 +6,34 @@ import docspell.query.ItemQuery._
object OperatorParser {
private[this] val Eq: P[Operator] =
P.char('=').void.map(_ => Operator.Eq)
P.char('=').as(Operator.Eq)
private[this] val Neq: P[Operator] =
P.string("!=").void.map(_ => Operator.Neq)
P.string("!=").as(Operator.Neq)
private[this] val Like: P[Operator] =
P.char(':').void.map(_ => Operator.Like)
P.char(':').as(Operator.Like)
private[this] val Gt: P[Operator] =
P.char('>').void.map(_ => Operator.Gt)
P.char('>').as(Operator.Gt)
private[this] val Lt: P[Operator] =
P.char('<').void.map(_ => Operator.Lt)
P.char('<').as(Operator.Lt)
private[this] val Gte: P[Operator] =
P.string(">=").map(_ => Operator.Gte)
P.string(">=").as(Operator.Gte)
private[this] val Lte: P[Operator] =
P.string("<=").map(_ => Operator.Lte)
P.string("<=").as(Operator.Lte)
val op: P[Operator] =
P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt))
private[this] val anyOp: P[TagOperator] =
P.char(':').map(_ => TagOperator.AnyMatch)
P.char(':').as(TagOperator.AnyMatch)
private[this] val allOp: P[TagOperator] =
P.char('=').map(_ => TagOperator.AllMatch)
P.char('=').as(TagOperator.AllMatch)
val tagOp: P[TagOperator] =
P.oneOf(List(anyOp, allOp))

12
modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala
View File

@ -17,7 +17,7 @@ object SimpleExprParser {
P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore)
private[this] val inOrOpDate =
P.eitherOr(op ~ DateParser.localDate, inOp *> DateParser.localDateOrMore)
P.eitherOr(op ~ DateParser.date, inOp *> DateParser.dateOrMore)
val stringExpr: P[Expr] =
(AttrParser.stringAttr ~ inOrOpStr).map {
@ -65,6 +65,12 @@ object SimpleExprParser {
CustomFieldMatch(name, op, value)
}
val inboxExpr: P[Expr.InboxExpr] =
(P.string("inbox:") *> BasicParser.bool).map(Expr.InboxExpr.apply)
val dirExpr: P[Expr.DirectionExpr] =
(P.string("incoming:") *> BasicParser.bool).map(Expr.DirectionExpr.apply)
val simpleExpr: P[Expr] =
P.oneOf(
List(
@ -75,7 +81,9 @@ object SimpleExprParser {
tagIdExpr,
tagExpr,
catExpr,
customFieldExpr
customFieldExpr,
inboxExpr,
dirExpr
)
)
}

51
modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala
View File

@ -2,35 +2,68 @@ package docspell.query.internal
import minitest._
import docspell.query.Date
import java.time.Period
object DateParserTest extends SimpleTestSuite {
def ld(year: Int, m: Int, d: Int): Date =
Date(year, m, d)
def ld(year: Int, m: Int, d: Int): Date.DateLiteral =
Date(year, m, d).fold(throw _, identity)
def ldPlus(year: Int, m: Int, d: Int, p: Period): Date.Calc =
Date.Calc(ld(year, m, d), Date.CalcDirection.Plus, p)
def ldMinus(year: Int, m: Int, d: Int, p: Period): Date.Calc =
Date.Calc(ld(year, m, d), Date.CalcDirection.Minus, p)
test("local date string") {
val p = DateParser.localDateFromString
val p = DateParser.dateFromString
assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22)))
assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11)))
assertEquals(p.parseAll("2032-01-21"), Right(ld(2032, 1, 21)))
assert(p.parseAll("0-0-0").isLeft)
assert(p.parseAll("2021-02-30").isRight)
assert(p.parseAll("2021-02-30").isLeft)
}
test("local date millis") {
val p = DateParser.dateFromMillis
assertEquals(p.parseAll("0"), Right(Date(0)))
assertEquals(p.parseAll("ms0"), Right(Date(0)))
assertEquals(
p.parseAll("1600000065463"),
p.parseAll("ms1600000065463"),
Right(Date(1600000065463L))
)
}
test("local date") {
val p = DateParser.localDate
val p = DateParser.date
assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22)))
assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11)))
assertEquals(p.parseAll("0"), Right(Date(0)))
assertEquals(p.parseAll("1600000065463"), Right(Date(1600000065463L)))
assertEquals(p.parseAll("ms0"), Right(Date(0)))
assertEquals(p.parseAll("ms1600000065463"), Right(Date(1600000065463L)))
}
test("local partial date") {
val p = DateParser.date
assertEquals(p.parseAll("2021-04"), Right(ld(2021, 4, 1)))
assertEquals(p.parseAll("2021-12"), Right(ld(2021, 12, 1)))
assert(p.parseAll("2021-13").isLeft)
assert(p.parseAll("2021-28").isLeft)
assertEquals(p.parseAll("2021"), Right(ld(2021, 1, 1)))
}
test("date calcs") {
val p = DateParser.date
assertEquals(p.parseAll("2020-02;+2d"), Right(ldPlus(2020, 2, 1, Period.ofDays(2))))
assertEquals(
p.parseAll("today;-2m"),
Right(Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2)))
)
}
test("period") {
val p = DateParser.periods
assertEquals(p.parseAll("15d"), Right(Period.ofDays(15)))
assertEquals(p.parseAll("15m"), Right(Period.ofMonths(15)))
assertEquals(p.parseAll("15d10m"), Right(Period.ofMonths(10).plus(Period.ofDays(15))))
assertEquals(p.parseAll("10m15d"), Right(Period.ofMonths(10).plus(Period.ofDays(15))))
}
}

19
modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala Normal file
View File

@ -0,0 +1,19 @@
package docspell.query.internal
import minitest._
import cats.parse.{Parser => P}
object MacroParserTest extends SimpleTestSuite {
test("fail with unkown macro names") {
val p = MacroParser.parser(Map.empty)
assert(p.parseAll("$bla:blup").isLeft) // TODO check error message
}
test("select correct parser") {
val p =
MacroParser.parser[Int](Map("one" -> P.anyChar.as(1), "two" -> P.anyChar.as(2)))
assertEquals(p.parseAll("$one:y"), Right(1))
assertEquals(p.parseAll("$two:y"), Right(2))
}
}

27
modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala
View File

@ -4,6 +4,7 @@ import cats.data.{NonEmptyList => Nel}
import docspell.query.ItemQuery._
import minitest._
import docspell.query.Date
import java.time.Period
object SimpleExprParserTest extends SimpleTestSuite {
@ -39,8 +40,8 @@ object SimpleExprParserTest extends SimpleTestSuite {
test("date expr") {
val p = SimpleExprParser.dateExpr
assertEquals(
p.parseAll("due:2021-03-14"),
Right(dateExpr(Operator.Like, Attr.DueDate, ld(2021, 3, 14)))
p.parseAll("date:2021-03-14"),
Right(dateExpr(Operator.Like, Attr.Date, ld(2021, 3, 14)))
)
assertEquals(
p.parseAll("due<2021-03-14"),
@ -50,6 +51,28 @@ object SimpleExprParserTest extends SimpleTestSuite {
p.parseAll("due~=2021-03-14,2021-03-13"),
Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13))))
)
assertEquals(
p.parseAll("due>2021"),
Right(dateExpr(Operator.Gt, Attr.DueDate, ld(2021, 1, 1)))
)
assertEquals(
p.parseAll("date<2021-01"),
Right(dateExpr(Operator.Lt, Attr.Date, ld(2021, 1, 1)))
)
assertEquals(
p.parseAll("date<today"),
Right(dateExpr(Operator.Lt, Attr.Date, Date.Today))
)
assertEquals(
p.parseAll("date>today;-2m"),
Right(
dateExpr(
Operator.Gt,
Attr.Date,
Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2))
)
)
)
}
test("exists expr") {

1
modules/restserver/src/main/resources/logback.xml
View File

@ -9,6 +9,7 @@
<logger name="docspell" level="debug" />
<logger name="emil" level="debug"/>
<logger name="docspell.store.queries.QItem" level="trace"/>
<root level="INFO">
<appender-ref ref="STDOUT" />

62
modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala
View File

@ -1,6 +1,7 @@
package docspell.store.qb.generator
import java.time.{Instant, LocalDate}
import java.time.Instant
import java.time.LocalDate
import cats.data.NonEmptyList
@ -9,27 +10,27 @@ import docspell.query.ItemQuery._
import docspell.query.{Date, ItemQuery}
import docspell.store.qb.DSL._
import docspell.store.qb.{Operator => QOp, _}
import docspell.store.queries.QueryWildcard
import docspell.store.records.{RCustomField, RCustomFieldValue, TagItemName}
import doobie.util.Put
import docspell.store.queries.QueryWildcard
object ItemQueryGenerator {
def apply(tables: Tables, coll: Ident)(q: ItemQuery)(implicit
def apply(today: LocalDate, tables: Tables, coll: Ident)(q: ItemQuery)(implicit
PT: Put[Timestamp]
): Condition =
fromExpr(tables, coll)(q.expr)
fromExpr(today, tables, coll)(q.expr)
final def fromExpr(tables: Tables, coll: Ident)(
final def fromExpr(today: LocalDate, tables: Tables, coll: Ident)(
expr: Expr
)(implicit PT: Put[Timestamp]): Condition =
expr match {
case Expr.AndExpr(inner) =>
Condition.And(inner.map(fromExpr(tables, coll)))
Condition.And(inner.map(fromExpr(today, tables, coll)))
case Expr.OrExpr(inner) =>
Condition.Or(inner.map(fromExpr(tables, coll)))
Condition.Or(inner.map(fromExpr(today, tables, coll)))
case Expr.NotExpr(inner) =>
inner match {
@ -71,7 +72,7 @@ object ItemQueryGenerator {
Condition.unit
case _ =>
Condition.Not(fromExpr(tables, coll)(inner))
Condition.Not(fromExpr(today, tables, coll)(inner))
}
case Expr.Exists(field) =>
@ -87,9 +88,10 @@ object ItemQueryGenerator {
}
case Expr.SimpleExpr(op, Property.DateProperty(attr, value)) =>
val dt = dateToTimestamp(value)
val col = timestampColumn(tables)(attr)
Condition.CompareVal(col, makeOp(op), dt)
val dt = dateToTimestamp(today)(value)
val col = timestampColumn(tables)(attr)
val noLikeOp = if (op == Operator.Like) Operator.Eq else op
Condition.CompareVal(col, makeOp(noLikeOp), dt)
case Expr.InExpr(attr, values) =>
val col = stringColumn(tables)(attr)
@ -98,10 +100,18 @@ object ItemQueryGenerator {
case Expr.InDateExpr(attr, values) =>
val col = timestampColumn(tables)(attr)
val dts = values.map(dateToTimestamp)
val dts = values.map(dateToTimestamp(today))
if (values.tail.isEmpty) col === dts.head
else col.in(dts)
case Expr.DirectionExpr(incoming) =>
if (incoming) tables.item.incoming === Direction.Incoming
else tables.item.incoming === Direction.Outgoing
case Expr.InboxExpr(flag) =>
if (flag) tables.item.state === ItemState.created
else tables.item.state === ItemState.confirmed
case Expr.TagIdsMatch(op, tags) =>
val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption)
NonEmptyList
@ -142,12 +152,31 @@ object ItemQueryGenerator {
Condition.unit
}
private def dateToTimestamp(date: Date): Timestamp =
private def dateToTimestamp(today: LocalDate)(date: Date): Timestamp =
date match {
case Date.Local(year, month, day) =>
Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay())
case d: Date.DateLiteral =>
val ld = dateLiteralToDate(today)(d)
println(s">>>> date= $ld")
Timestamp.atUtc(ld.atStartOfDay)
case Date.Calc(date, c, period) =>
val ld = c match {
case Date.CalcDirection.Plus =>
dateLiteralToDate(today)(date).plus(period)
case Date.CalcDirection.Minus =>
dateLiteralToDate(today)(date).minus(period)
}
println(s">>>> date= $ld")
Timestamp.atUtc(ld.atStartOfDay())
}
private def dateLiteralToDate(today: LocalDate)(dateLit: Date.DateLiteral): LocalDate =
dateLit match {
case Date.Local(date) =>
date
case Date.Millis(ms) =>
Timestamp(Instant.ofEpochMilli(ms))
Instant.ofEpochMilli(ms).atZone(Timestamp.UTC).toLocalDate()
case Date.Today =>
today
}
private def anyColumn(tables: Tables)(attr: Attr): Column[_] =
@ -171,6 +200,7 @@ object ItemQueryGenerator {
case Attr.ItemId => tables.item.id.cast[String]
case Attr.ItemName => tables.item.name
case Attr.ItemSource => tables.item.source
case Attr.ItemNotes => tables.item.notes
case Attr.Correspondent.OrgId => tables.corrOrg.oid.cast[String]
case Attr.Correspondent.OrgName => tables.corrOrg.name
case Attr.Correspondent.PersonId => tables.corrPers.pid.cast[String]

39
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -1,5 +1,7 @@
package docspell.store.queries
import java.time.LocalDate
import cats.data.{NonEmptyList => Nel}
import cats.effect.Sync
import cats.effect.concurrent.Ref
@ -226,41 +228,42 @@ object QItem {
findCustomFieldValuesForColl(coll, q.customValues)
.map(itemIds => i.id.in(itemIds))
def queryCondFromExpr(coll: Ident, q: ItemQuery): Condition = {
def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery): Condition = {
val tables = Tables(i, org, pers0, pers1, equip, f, a, m)
ItemQueryGenerator.fromExpr(tables, coll)(q.expr)
ItemQueryGenerator.fromExpr(today, tables, coll)(q.expr)
}
def queryCondition(coll: Ident, cond: Query.QueryCond): Condition =
def queryCondition(today: LocalDate, coll: Ident, cond: Query.QueryCond): Condition =
cond match {
case fm: Query.QueryForm =>
queryCondFromForm(coll, fm)
case expr: Query.QueryExpr =>
queryCondFromExpr(coll, expr.q)
queryCondFromExpr(today, coll, expr.q)
}
def findItems(
q: Query,
today: LocalDate,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] = {
val sql = findItemsBase(q.fix, maxNoteLen)
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
.limit(batch)
.build
logger.trace(s"List $batch items: $sql")
sql.query[ListItem].stream
}
def searchStats(q: Query): ConnectionIO[SearchSummary] =
def searchStats(today: LocalDate)(q: Query): ConnectionIO[SearchSummary] =
for {
count <- searchCountSummary(q)
tags <- searchTagSummary(q)
fields <- searchFieldSummary(q)
folders <- searchFolderSummary(q)
count <- searchCountSummary(today)(q)
tags <- searchTagSummary(today)(q)
fields <- searchFieldSummary(today)(q)
folders <- searchFolderSummary(today)(q)
} yield SearchSummary(count, tags, fields, folders)
def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = {
def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = {
val tagFrom =
from(ti)
.innerJoin(tag, tag.tid === ti.tagId)
@ -270,7 +273,7 @@ object QItem {
findItemsBase(q.fix, 0).unwrap
.withSelect(select(tag.all).append(count(i.id).as("num")))
.changeFrom(_.prepend(tagFrom))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
.groupBy(tag.tid)
.build
.query[TagCount]
@ -284,27 +287,27 @@ object QItem {
} yield existing ++ other.map(TagCount(_, 0))
}
def searchCountSummary(q: Query): ConnectionIO[Int] =
def searchCountSummary(today: LocalDate)(q: Query): ConnectionIO[Int] =
findItemsBase(q.fix, 0).unwrap
.withSelect(Nel.of(count(i.id).as("num")))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
.build
.query[Int]
.unique
def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = {
def searchFolderSummary(today: LocalDate)(q: Query): ConnectionIO[List[FolderCount]] = {
val fu = RUser.as("fu")
findItemsBase(q.fix, 0).unwrap
.withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num")))
.changeFrom(_.innerJoin(fu, fu.uid === f.owner))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
.groupBy(f.id, f.name, f.owner, fu.login)
.build
.query[FolderCount]
.to[List]
}
def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = {
def searchFieldSummary(today: LocalDate)(q: Query): ConnectionIO[List[FieldStats]] = {
val fieldJoin =
from(cv)
.innerJoin(cf, cf.id === cv.field)
@ -313,7 +316,7 @@ object QItem {
val base =
findItemsBase(q.fix, 0).unwrap
.changeFrom(_.prepend(fieldJoin))
.changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
.groupBy(GroupBy(cf.all))
val basicFields = Nel.of(

28
modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala
View File

@ -22,11 +22,12 @@ object ItemQueryGeneratorTest extends SimpleTestSuite {
RAttachment.as("a"),
RAttachmentMeta.as("m")
)
val now: LocalDate = LocalDate.of(2021, 2, 25)
test("basic test") {
val q = ItemQueryParser
.parseUnsafe("(& name:hello date>=2020-02-01 (| source=expense folder=test ))")
val cond = ItemQueryGenerator(tables, Ident.unsafe("coll"))(q)
val cond = ItemQueryGenerator(now, tables, Ident.unsafe("coll"))(q)
val expect =
tables.item.name.like("hello") && tables.item.itemDate >= Timestamp.atUtc(
LocalDate.of(2020, 2, 1).atStartOfDay()
@ -35,29 +36,4 @@ object ItemQueryGeneratorTest extends SimpleTestSuite {
assertEquals(cond, expect)
}
// test("migration2") {
// withStore("db2") { store =>
// val c = RCollective(
// Ident.unsafe("coll1"),
// CollectiveState.Active,
// Language.German,
// true,
// Timestamp.Epoch
// )
// val e =
// REquipment(
// Ident.unsafe("equip"),
// Ident.unsafe("coll1"),
// "name",
// Timestamp.Epoch,
// Timestamp.Epoch,
// None
// )
//
// for {
// _ <- store.transact(RCollective.insert(c))
// _ <- store.transact(REquipment.insert(e)).map(_ => ())
// } yield ()
// }
// }
}

4
project/Dependencies.scala
View File

@ -33,6 +33,7 @@ object Dependencies {
val PoiVersion = "4.1.2"
val PostgresVersion = "42.2.19"
val PureConfigVersion = "0.14.1"
val ScalaJavaTimeVersion = "2.2.0"
val Slf4jVersion = "1.7.30"
val StanfordNlpVersion = "4.2.0"
val TikaVersion = "1.25"
@ -54,6 +55,9 @@ object Dependencies {
val catsJS = Def.setting("org.typelevel" %%% "cats-core" % "2.4.2")
val scalaJavaTime =
Def.setting("io.github.cquiroz" %%% "scala-java-time" % ScalaJavaTimeVersion)
val kittens = Seq(
"org.typelevel" %% "kittens" % KittensVersion
)