From 9013d9264ea013f30bd84616b40ce4a92b4e2e43 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 28 Feb 2021 16:11:25 +0100 Subject: [PATCH] Add more convenient date parsers and some basic macros --- build.sbt | 4 +- .../docspell/backend/ops/OFulltext.scala | 6 +- .../docspell/backend/ops/OItemSearch.scala | 44 ++++++---- .../joex/notify/NotifyDueItemsTask.scala | 6 +- .../src/main/scala/docspell/query/Date.scala | 32 +++++-- .../main/scala/docspell/query/ItemQuery.scala | 23 +++++ .../docspell/query/ItemQueryParser.scala | 2 +- .../docspell/query/internal/AttrParser.scala | 34 ++++---- .../docspell/query/internal/BasicParser.scala | 6 ++ .../docspell/query/internal/DateParser.scala | 85 ++++++++++++++++--- .../docspell/query/internal/ExprParser.scala | 9 +- .../docspell/query/internal/ExprUtil.scala | 10 +++ .../docspell/query/internal/MacroParser.scala | 65 ++++++++++++++ .../query/internal/OperatorParser.scala | 18 ++-- .../query/internal/SimpleExprParser.scala | 12 ++- .../query/internal/DateParserTest.scala | 51 +++++++++-- .../query/internal/MacroParserTest.scala | 19 +++++ .../query/internal/SimpleExprParserTest.scala | 27 +++++- .../restserver/src/main/resources/logback.xml | 1 + .../qb/generator/ItemQueryGenerator.scala | 62 ++++++++++---- .../scala/docspell/store/queries/QItem.scala | 39 +++++---- .../generator/ItemQueryGeneratorTest.scala | 28 +----- project/Dependencies.scala | 4 + 23 files changed, 445 insertions(+), 142 deletions(-) create mode 100644 modules/query/src/main/scala/docspell/query/internal/MacroParser.scala create mode 100644 modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala diff --git a/build.sbt b/build.sbt index 351f40d7..884234f3 100644 --- a/build.sbt +++ b/build.sbt @@ -275,7 +275,9 @@ val query = .settings( name := "docspell-query", libraryDependencies += - Dependencies.catsParseJS.value + Dependencies.catsParseJS.value, + libraryDependencies += + Dependencies.scalaJavaTime.value ) .jsSettings( Test / fork := false diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 52197e17..0dd2348c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -159,13 +159,14 @@ object OFulltext { for { folder <- store.transact(QFolder.getMemberFolders(account)) + now <- Timestamp.current[F] itemIds <- fts .searchAll(fq.withFolders(folder)) .flatMap(r => Stream.emits(r.results.map(_.itemId))) .compile .to(Set) q = Query.empty(account).withFix(_.copy(itemIds = itemIds.some)) - res <- store.transact(QItem.searchStats(q)) + res <- store.transact(QItem.searchStats(now.toUtcDate)(q)) } yield res } @@ -221,7 +222,8 @@ object OFulltext { .compile .to(Set) qnext = q.withFix(_.copy(itemIds = items.some)) - res <- store.transact(QItem.searchStats(qnext)) + now <- Timestamp.current[F] + res <- store.transact(QItem.searchStats(now.toUtcDate)(qnext)) } yield res // Helper diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 724ee18e..a74e451a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -127,27 +127,39 @@ object OItemSearch { .map(opt => opt.flatMap(_.filterCollective(collective))) def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] = - store - .transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong)) - .compile - .toVector + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap { today => + store + .transact( + QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong) + ) + .compile + .toVector + } def findItemsWithTags( maxNoteLen: Int - )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = { - val search = QItem.findItems(q, maxNoteLen: Int, batch) - store - .transact( - QItem - .findItemsWithTags(q.fix.account.collective, search) - .take(batch.limit.toLong) - ) - .compile - .toVector - } + )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = + for { + now <- Timestamp.current[F] + search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch) + res <- store + .transact( + QItem + .findItemsWithTags(q.fix.account.collective, search) + .take(batch.limit.toLong) + ) + .compile + .toVector + } yield res def findItemsSummary(q: Query): F[SearchSummary] = - store.transact(QItem.searchStats(q)) + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap(today => store.transact(QItem.searchStats(today)(q))) def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index 1000d630..4ce26507 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -85,7 +85,11 @@ object NotifyDueItemsTask { ) res <- ctx.store - .transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong)) + .transact( + QItem + .findItems(q, now.toUtcDate, 0, Batch.limit(maxItems)) + .take(maxItems.toLong) + ) .compile .toVector } yield res diff --git a/modules/query/src/main/scala/docspell/query/Date.scala b/modules/query/src/main/scala/docspell/query/Date.scala index 30e9dabb..21ce9b35 100644 --- a/modules/query/src/main/scala/docspell/query/Date.scala +++ b/modules/query/src/main/scala/docspell/query/Date.scala @@ -1,14 +1,32 @@ package docspell.query -sealed trait Date -object Date { - def apply(y: Int, m: Int, d: Int): Date = - Local(y, m, d) +import java.time.LocalDate +import java.time.Period - def apply(ms: Long): Date = +import cats.implicits._ + +sealed trait Date + +object Date { + def apply(y: Int, m: Int, d: Int): Either[Throwable, DateLiteral] = + Either.catchNonFatal(Local(LocalDate.of(y, m, d))) + + def apply(ms: Long): DateLiteral = Millis(ms) - final case class Local(year: Int, month: Int, day: Int) extends Date + sealed trait DateLiteral extends Date - final case class Millis(ms: Long) extends Date + final case class Local(date: LocalDate) extends DateLiteral + + final case class Millis(ms: Long) extends DateLiteral + + case object Today extends DateLiteral + + sealed trait CalcDirection + object CalcDirection { + case object Plus extends CalcDirection + case object Minus extends CalcDirection + } + + case class Calc(date: DateLiteral, calc: CalcDirection, period: Period) extends Date } diff --git a/modules/query/src/main/scala/docspell/query/ItemQuery.scala b/modules/query/src/main/scala/docspell/query/ItemQuery.scala index 46b9e051..ec824ecb 100644 --- a/modules/query/src/main/scala/docspell/query/ItemQuery.scala +++ b/modules/query/src/main/scala/docspell/query/ItemQuery.scala @@ -40,6 +40,7 @@ object ItemQuery { case object ItemName extends StringAttr case object ItemSource extends StringAttr + case object ItemNotes extends StringAttr case object ItemId extends StringAttr case object Date extends DateAttr case object DueDate extends DateAttr @@ -69,6 +70,11 @@ object ItemQuery { final case class StringProperty(attr: StringAttr, value: String) extends Property final case class DateProperty(attr: DateAttr, value: Date) extends Property + def apply(sa: StringAttr, value: String): Property = + StringProperty(sa, value) + + def apply(da: DateAttr, value: Date): Property = + DateProperty(da, value) } sealed trait Expr { @@ -88,6 +94,8 @@ object ItemQuery { final case class Exists(field: Attr) extends Expr final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr + final case class InboxExpr(inbox: Boolean) extends Expr + final case class DirectionExpr(incoming: Boolean) extends Expr final case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr final case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr @@ -97,6 +105,21 @@ object ItemQuery { extends Expr final case class Fulltext(query: String) extends Expr + + def or(expr0: Expr, exprs: Expr*): OrExpr = + OrExpr(Nel.of(expr0, exprs: _*)) + + def and(expr0: Expr, exprs: Expr*): AndExpr = + AndExpr(Nel.of(expr0, exprs: _*)) + + def string(op: Operator, attr: StringAttr, value: String): SimpleExpr = + SimpleExpr(op, Property(attr, value)) + + def like(attr: StringAttr, value: String): SimpleExpr = + string(Operator.Like, attr, value) + + def date(op: Operator, attr: DateAttr, value: Date): SimpleExpr = + SimpleExpr(op, Property(attr, value)) } } diff --git a/modules/query/src/main/scala/docspell/query/ItemQueryParser.scala b/modules/query/src/main/scala/docspell/query/ItemQueryParser.scala index 0a7b8d81..cea6c09e 100644 --- a/modules/query/src/main/scala/docspell/query/ItemQueryParser.scala +++ b/modules/query/src/main/scala/docspell/query/ItemQueryParser.scala @@ -16,7 +16,7 @@ object ItemQueryParser { ExprParser .parseQuery(in) .left - .map(pe => s"Error parsing: '$input': $pe") + .map(pe => s"Error parsing: '$input': $pe") //TODO .map(q => q.copy(expr = ExprUtil.reduce(q.expr))) } diff --git a/modules/query/src/main/scala/docspell/query/internal/AttrParser.scala b/modules/query/src/main/scala/docspell/query/internal/AttrParser.scala index f9520a61..d5289c67 100644 --- a/modules/query/src/main/scala/docspell/query/internal/AttrParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/AttrParser.scala @@ -7,57 +7,60 @@ import docspell.query.ItemQuery.Attr object AttrParser { val name: P[Attr.StringAttr] = - P.ignoreCase("name").map(_ => Attr.ItemName) + P.ignoreCase("name").as(Attr.ItemName) val source: P[Attr.StringAttr] = - P.ignoreCase("source").map(_ => Attr.ItemSource) + P.ignoreCase("source").as(Attr.ItemSource) val id: P[Attr.StringAttr] = - P.ignoreCase("id").map(_ => Attr.ItemId) + P.ignoreCase("id").as(Attr.ItemId) val date: P[Attr.DateAttr] = - P.ignoreCase("date").map(_ => Attr.Date) + P.ignoreCase("date").as(Attr.Date) + + val notes: P[Attr.StringAttr] = + P.ignoreCase("notes").as(Attr.ItemNotes) val dueDate: P[Attr.DateAttr] = - P.stringIn(List("dueDate", "due", "due-date")).map(_ => Attr.DueDate) + P.stringIn(List("dueDate", "due", "due-date")).as(Attr.DueDate) val corrOrgId: P[Attr.StringAttr] = P.stringIn(List("correspondent.org.id", "corr.org.id")) - .map(_ => Attr.Correspondent.OrgId) + .as(Attr.Correspondent.OrgId) val corrOrgName: P[Attr.StringAttr] = P.stringIn(List("correspondent.org.name", "corr.org.name")) - .map(_ => Attr.Correspondent.OrgName) + .as(Attr.Correspondent.OrgName) val corrPersId: P[Attr.StringAttr] = P.stringIn(List("correspondent.person.id", "corr.pers.id")) - .map(_ => Attr.Correspondent.PersonId) + .as(Attr.Correspondent.PersonId) val corrPersName: P[Attr.StringAttr] = P.stringIn(List("correspondent.person.name", "corr.pers.name")) - .map(_ => Attr.Correspondent.PersonName) + .as(Attr.Correspondent.PersonName) val concPersId: P[Attr.StringAttr] = P.stringIn(List("concerning.person.id", "conc.pers.id")) - .map(_ => Attr.Concerning.PersonId) + .as(Attr.Concerning.PersonId) val concPersName: P[Attr.StringAttr] = P.stringIn(List("concerning.person.name", "conc.pers.name")) - .map(_ => Attr.Concerning.PersonName) + .as(Attr.Concerning.PersonName) val concEquipId: P[Attr.StringAttr] = P.stringIn(List("concerning.equip.id", "conc.equip.id")) - .map(_ => Attr.Concerning.EquipId) + .as(Attr.Concerning.EquipId) val concEquipName: P[Attr.StringAttr] = P.stringIn(List("concerning.equip.name", "conc.equip.name")) - .map(_ => Attr.Concerning.EquipName) + .as(Attr.Concerning.EquipName) val folderId: P[Attr.StringAttr] = - P.ignoreCase("folder.id").map(_ => Attr.Folder.FolderId) + P.ignoreCase("folder.id").as(Attr.Folder.FolderId) val folderName: P[Attr.StringAttr] = - P.ignoreCase("folder").map(_ => Attr.Folder.FolderName) + P.ignoreCase("folder").as(Attr.Folder.FolderName) val dateAttr: P[Attr.DateAttr] = P.oneOf(List(date, dueDate)) @@ -68,6 +71,7 @@ object AttrParser { name, source, id, + notes, corrOrgId, corrOrgName, corrPersId, diff --git a/modules/query/src/main/scala/docspell/query/internal/BasicParser.scala b/modules/query/src/main/scala/docspell/query/internal/BasicParser.scala index ca2c3462..c4edd070 100644 --- a/modules/query/src/main/scala/docspell/query/internal/BasicParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/BasicParser.scala @@ -38,4 +38,10 @@ object BasicParser { val stringOrMore: P[Nel[String]] = singleString.repSep(stringListSep) + val bool: P[Boolean] = { + val trueP = P.stringIn(List("yes", "true", "Yes", "True")).as(true) + val falseP = P.stringIn(List("no", "false", "No", "False")).as(false) + trueP | falseP + } + } diff --git a/modules/query/src/main/scala/docspell/query/internal/DateParser.scala b/modules/query/src/main/scala/docspell/query/internal/DateParser.scala index 25d1dc1c..33e0d556 100644 --- a/modules/query/src/main/scala/docspell/query/internal/DateParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/DateParser.scala @@ -1,7 +1,8 @@ package docspell.query.internal -import cats.data.NonEmptyList -import cats.implicits._ +import java.time.Period + +import cats.data.{NonEmptyList => Nel} import cats.parse.{Numbers, Parser => P} import docspell.query.Date @@ -26,21 +27,79 @@ object DateParser { digits2.filter(n => n >= 1 && n <= 31) private val dateSep: P[Unit] = - P.anyChar.void + P.charIn('-', '/').void - val localDateFromString: P[Date] = - ((digits4 <* dateSep) ~ (month <* dateSep) ~ day).mapFilter { - case ((year, month), day) => - Either.catchNonFatal(Date(year, month, day)).toOption + private val dateString: P[((Int, Option[Int]), Option[Int])] = + digits4 ~ (dateSep *> month).? ~ (dateSep *> day).? + + private[internal] val dateFromString: P[Date.DateLiteral] = + dateString.mapFilter { case ((year, month), day) => + Date(year, month.getOrElse(1), day.getOrElse(1)).toOption } - val dateFromMillis: P[Date] = - longParser.map(Date.apply) + private[internal] val dateFromMillis: P[Date.DateLiteral] = + P.string("ms") *> longParser.map(Date.apply) - val localDate: P[Date] = - localDateFromString.backtrack.orElse(dateFromMillis) + private val dateFromToday: P[Date.DateLiteral] = + P.string("today").as(Date.Today) - val localDateOrMore: P[NonEmptyList[Date]] = - localDate.repSep(BasicParser.stringListSep) + val dateLiteral: P[Date.DateLiteral] = + P.oneOf(List(dateFromString, dateFromToday, dateFromMillis)) + // val dateLiteralOrMore: P[NonEmptyList[Date.DateLiteral]] = + // dateLiteral.repSep(BasicParser.stringListSep) + + val dateCalcDirection: P[Date.CalcDirection] = + P.oneOf( + List( + P.char('+').as(Date.CalcDirection.Plus), + P.char('-').as(Date.CalcDirection.Minus) + ) + ) + + def periodPart(unitSuffix: Char, f: Int => Period): P[Period] = + ((Numbers.nonZeroDigit ~ Numbers.digits0).void.string.soft <* P.ignoreCaseChar( + unitSuffix + )) + .map(n => f(n.toInt)) + + private[this] val periodMonths: P[Period] = + periodPart('m', n => Period.ofMonths(n)) + + private[this] val periodDays: P[Period] = + periodPart('d', n => Period.ofDays(n)) + + val period: P[Period] = + periodDays.eitherOr(periodMonths).map(_.fold(identity, identity)) + + val periods: P[Period] = + period.rep.map(_.reduceLeft((p0, p1) => p0.plus(p1))) + + val dateRange: P[(Date, Date)] = + ((dateLiteral <* P.char(';')) ~ dateCalcDirection.eitherOr(P.char('/')) ~ period) + .map { case ((date, calc), period) => + calc match { + case Right(Date.CalcDirection.Plus) => + (date, Date.Calc(date, Date.CalcDirection.Plus, period)) + case Right(Date.CalcDirection.Minus) => + (Date.Calc(date, Date.CalcDirection.Minus, period), date) + case Left(_) => + ( + Date.Calc(date, Date.CalcDirection.Minus, period), + Date.Calc(date, Date.CalcDirection.Plus, period) + ) + } + } + + val date: P[Date] = + (dateLiteral ~ (P.char(';') *> dateCalcDirection ~ period).?).map { + case (date, Some((c, p))) => + Date.Calc(date, c, p) + + case (date, None) => + date + } + + val dateOrMore: P[Nel[Date]] = + date.repSep(BasicParser.stringListSep) } diff --git a/modules/query/src/main/scala/docspell/query/internal/ExprParser.scala b/modules/query/src/main/scala/docspell/query/internal/ExprParser.scala index 693c087d..329ec030 100644 --- a/modules/query/src/main/scala/docspell/query/internal/ExprParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/ExprParser.scala @@ -24,10 +24,11 @@ object ExprParser { val exprParser: P[Expr] = P.recursive[Expr] { recurse => - val andP = and(recurse) - val orP = or(recurse) - val notP = not(recurse) - P.oneOf(SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil) + val andP = and(recurse) + val orP = or(recurse) + val notP = not(recurse) + val macros = MacroParser.all + P.oneOf(SimpleExprParser.simpleExpr :: macros :: andP :: orP :: notP :: Nil) } def parseQuery(input: String): Either[P.Error, ItemQuery] = { diff --git a/modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala b/modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala index 082f2b28..75007d11 100644 --- a/modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala +++ b/modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala @@ -22,10 +22,20 @@ object ExprUtil { inner match { case NotExpr(inner2) => reduce(inner2) + case InboxExpr(flag) => + InboxExpr(!flag) + case DirectionExpr(flag) => + DirectionExpr(!flag) case _ => expr } + case DirectionExpr(_) => + expr + + case InboxExpr(_) => + expr + case InExpr(_, _) => expr diff --git a/modules/query/src/main/scala/docspell/query/internal/MacroParser.scala b/modules/query/src/main/scala/docspell/query/internal/MacroParser.scala new file mode 100644 index 00000000..2a2c2d41 --- /dev/null +++ b/modules/query/src/main/scala/docspell/query/internal/MacroParser.scala @@ -0,0 +1,65 @@ +package docspell.query.internal + +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery._ + +object MacroParser { + private[this] val macroDef: P[String] = + P.char('$') *> BasicParser.identParser <* P.char(':') + + def parser[A](macros: Map[String, P[A]]): P[A] = { + val p: P[P[A]] = macroDef.map { name => + macros + .get(name) + .getOrElse(P.failWith(s"Unknown macro: $name")) + } + + val px = (p ~ P.index ~ BasicParser.singleString).map { case ((pexpr, index), str) => + pexpr + .parseAll(str) + .left + .map(err => err.copy(failedAtOffset = err.failedAtOffset + index)) + } + + P.select(px)(P.Fail) + } + + // --- definitions of available macros + + /** Expands in an OR expression that matches name fields of item and + * correspondent/concerning metadata. + */ + val names: P[Expr] = + P.string(P.anyChar.rep.void).map { input => + Expr.or( + Expr.like(Attr.ItemName, input), + Expr.like(Attr.ItemNotes, input), + Expr.like(Attr.Correspondent.OrgName, input), + Expr.like(Attr.Correspondent.PersonName, input), + Expr.like(Attr.Concerning.PersonName, input), + Expr.like(Attr.Concerning.EquipName, input) + ) + } + + def dateRange(attr: Attr.DateAttr): P[Expr] = + DateParser.dateRange.map { case (left, right) => + Expr.and( + Expr.date(Operator.Gte, attr, left), + Expr.date(Operator.Lte, attr, right) + ) + } + + // --- all macro parser + + val allMacros: Map[String, P[Expr]] = + Map( + "names" -> names, + "datein" -> dateRange(Attr.Date), + "duein" -> dateRange(Attr.DueDate) + ) + + val all: P[Expr] = + parser(allMacros) + +} diff --git a/modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala b/modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala index f130ed8d..de93fb4f 100644 --- a/modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala @@ -6,34 +6,34 @@ import docspell.query.ItemQuery._ object OperatorParser { private[this] val Eq: P[Operator] = - P.char('=').void.map(_ => Operator.Eq) + P.char('=').as(Operator.Eq) private[this] val Neq: P[Operator] = - P.string("!=").void.map(_ => Operator.Neq) + P.string("!=").as(Operator.Neq) private[this] val Like: P[Operator] = - P.char(':').void.map(_ => Operator.Like) + P.char(':').as(Operator.Like) private[this] val Gt: P[Operator] = - P.char('>').void.map(_ => Operator.Gt) + P.char('>').as(Operator.Gt) private[this] val Lt: P[Operator] = - P.char('<').void.map(_ => Operator.Lt) + P.char('<').as(Operator.Lt) private[this] val Gte: P[Operator] = - P.string(">=").map(_ => Operator.Gte) + P.string(">=").as(Operator.Gte) private[this] val Lte: P[Operator] = - P.string("<=").map(_ => Operator.Lte) + P.string("<=").as(Operator.Lte) val op: P[Operator] = P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt)) private[this] val anyOp: P[TagOperator] = - P.char(':').map(_ => TagOperator.AnyMatch) + P.char(':').as(TagOperator.AnyMatch) private[this] val allOp: P[TagOperator] = - P.char('=').map(_ => TagOperator.AllMatch) + P.char('=').as(TagOperator.AllMatch) val tagOp: P[TagOperator] = P.oneOf(List(anyOp, allOp)) diff --git a/modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala b/modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala index 3b23ceea..f76fe947 100644 --- a/modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala +++ b/modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala @@ -17,7 +17,7 @@ object SimpleExprParser { P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore) private[this] val inOrOpDate = - P.eitherOr(op ~ DateParser.localDate, inOp *> DateParser.localDateOrMore) + P.eitherOr(op ~ DateParser.date, inOp *> DateParser.dateOrMore) val stringExpr: P[Expr] = (AttrParser.stringAttr ~ inOrOpStr).map { @@ -65,6 +65,12 @@ object SimpleExprParser { CustomFieldMatch(name, op, value) } + val inboxExpr: P[Expr.InboxExpr] = + (P.string("inbox:") *> BasicParser.bool).map(Expr.InboxExpr.apply) + + val dirExpr: P[Expr.DirectionExpr] = + (P.string("incoming:") *> BasicParser.bool).map(Expr.DirectionExpr.apply) + val simpleExpr: P[Expr] = P.oneOf( List( @@ -75,7 +81,9 @@ object SimpleExprParser { tagIdExpr, tagExpr, catExpr, - customFieldExpr + customFieldExpr, + inboxExpr, + dirExpr ) ) } diff --git a/modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala b/modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala index 281fc470..ff538363 100644 --- a/modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala +++ b/modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala @@ -2,35 +2,68 @@ package docspell.query.internal import minitest._ import docspell.query.Date +import java.time.Period object DateParserTest extends SimpleTestSuite { - def ld(year: Int, m: Int, d: Int): Date = - Date(year, m, d) + def ld(year: Int, m: Int, d: Int): Date.DateLiteral = + Date(year, m, d).fold(throw _, identity) + + def ldPlus(year: Int, m: Int, d: Int, p: Period): Date.Calc = + Date.Calc(ld(year, m, d), Date.CalcDirection.Plus, p) + + def ldMinus(year: Int, m: Int, d: Int, p: Period): Date.Calc = + Date.Calc(ld(year, m, d), Date.CalcDirection.Minus, p) test("local date string") { - val p = DateParser.localDateFromString + val p = DateParser.dateFromString assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22))) assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11))) assertEquals(p.parseAll("2032-01-21"), Right(ld(2032, 1, 21))) assert(p.parseAll("0-0-0").isLeft) - assert(p.parseAll("2021-02-30").isRight) + assert(p.parseAll("2021-02-30").isLeft) } test("local date millis") { val p = DateParser.dateFromMillis - assertEquals(p.parseAll("0"), Right(Date(0))) + assertEquals(p.parseAll("ms0"), Right(Date(0))) assertEquals( - p.parseAll("1600000065463"), + p.parseAll("ms1600000065463"), Right(Date(1600000065463L)) ) } test("local date") { - val p = DateParser.localDate + val p = DateParser.date assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22))) assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11))) - assertEquals(p.parseAll("0"), Right(Date(0))) - assertEquals(p.parseAll("1600000065463"), Right(Date(1600000065463L))) + assertEquals(p.parseAll("ms0"), Right(Date(0))) + assertEquals(p.parseAll("ms1600000065463"), Right(Date(1600000065463L))) + } + + test("local partial date") { + val p = DateParser.date + assertEquals(p.parseAll("2021-04"), Right(ld(2021, 4, 1))) + assertEquals(p.parseAll("2021-12"), Right(ld(2021, 12, 1))) + assert(p.parseAll("2021-13").isLeft) + assert(p.parseAll("2021-28").isLeft) + assertEquals(p.parseAll("2021"), Right(ld(2021, 1, 1))) + } + + test("date calcs") { + val p = DateParser.date + assertEquals(p.parseAll("2020-02;+2d"), Right(ldPlus(2020, 2, 1, Period.ofDays(2)))) + assertEquals( + p.parseAll("today;-2m"), + Right(Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2))) + ) + } + + test("period") { + val p = DateParser.periods + assertEquals(p.parseAll("15d"), Right(Period.ofDays(15))) + assertEquals(p.parseAll("15m"), Right(Period.ofMonths(15))) + assertEquals(p.parseAll("15d10m"), Right(Period.ofMonths(10).plus(Period.ofDays(15)))) + assertEquals(p.parseAll("10m15d"), Right(Period.ofMonths(10).plus(Period.ofDays(15)))) } } diff --git a/modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala b/modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala new file mode 100644 index 00000000..0836884a --- /dev/null +++ b/modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala @@ -0,0 +1,19 @@ +package docspell.query.internal + +import minitest._ +import cats.parse.{Parser => P} + +object MacroParserTest extends SimpleTestSuite { + + test("fail with unkown macro names") { + val p = MacroParser.parser(Map.empty) + assert(p.parseAll("$bla:blup").isLeft) // TODO check error message + } + + test("select correct parser") { + val p = + MacroParser.parser[Int](Map("one" -> P.anyChar.as(1), "two" -> P.anyChar.as(2))) + assertEquals(p.parseAll("$one:y"), Right(1)) + assertEquals(p.parseAll("$two:y"), Right(2)) + } +} diff --git a/modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala b/modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala index f3ee0ae5..9f79aa51 100644 --- a/modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala +++ b/modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala @@ -4,6 +4,7 @@ import cats.data.{NonEmptyList => Nel} import docspell.query.ItemQuery._ import minitest._ import docspell.query.Date +import java.time.Period object SimpleExprParserTest extends SimpleTestSuite { @@ -39,8 +40,8 @@ object SimpleExprParserTest extends SimpleTestSuite { test("date expr") { val p = SimpleExprParser.dateExpr assertEquals( - p.parseAll("due:2021-03-14"), - Right(dateExpr(Operator.Like, Attr.DueDate, ld(2021, 3, 14))) + p.parseAll("date:2021-03-14"), + Right(dateExpr(Operator.Like, Attr.Date, ld(2021, 3, 14))) ) assertEquals( p.parseAll("due<2021-03-14"), @@ -50,6 +51,28 @@ object SimpleExprParserTest extends SimpleTestSuite { p.parseAll("due~=2021-03-14,2021-03-13"), Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13)))) ) + assertEquals( + p.parseAll("due>2021"), + Right(dateExpr(Operator.Gt, Attr.DueDate, ld(2021, 1, 1))) + ) + assertEquals( + p.parseAll("date<2021-01"), + Right(dateExpr(Operator.Lt, Attr.Date, ld(2021, 1, 1))) + ) + assertEquals( + p.parseAll("datetoday;-2m"), + Right( + dateExpr( + Operator.Gt, + Attr.Date, + Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2)) + ) + ) + ) } test("exists expr") { diff --git a/modules/restserver/src/main/resources/logback.xml b/modules/restserver/src/main/resources/logback.xml index f9b2d921..d972abcb 100644 --- a/modules/restserver/src/main/resources/logback.xml +++ b/modules/restserver/src/main/resources/logback.xml @@ -9,6 +9,7 @@ + diff --git a/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala index 2e2b3233..173e8399 100644 --- a/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala +++ b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala @@ -1,6 +1,7 @@ package docspell.store.qb.generator -import java.time.{Instant, LocalDate} +import java.time.Instant +import java.time.LocalDate import cats.data.NonEmptyList @@ -9,27 +10,27 @@ import docspell.query.ItemQuery._ import docspell.query.{Date, ItemQuery} import docspell.store.qb.DSL._ import docspell.store.qb.{Operator => QOp, _} +import docspell.store.queries.QueryWildcard import docspell.store.records.{RCustomField, RCustomFieldValue, TagItemName} import doobie.util.Put -import docspell.store.queries.QueryWildcard object ItemQueryGenerator { - def apply(tables: Tables, coll: Ident)(q: ItemQuery)(implicit + def apply(today: LocalDate, tables: Tables, coll: Ident)(q: ItemQuery)(implicit PT: Put[Timestamp] ): Condition = - fromExpr(tables, coll)(q.expr) + fromExpr(today, tables, coll)(q.expr) - final def fromExpr(tables: Tables, coll: Ident)( + final def fromExpr(today: LocalDate, tables: Tables, coll: Ident)( expr: Expr )(implicit PT: Put[Timestamp]): Condition = expr match { case Expr.AndExpr(inner) => - Condition.And(inner.map(fromExpr(tables, coll))) + Condition.And(inner.map(fromExpr(today, tables, coll))) case Expr.OrExpr(inner) => - Condition.Or(inner.map(fromExpr(tables, coll))) + Condition.Or(inner.map(fromExpr(today, tables, coll))) case Expr.NotExpr(inner) => inner match { @@ -71,7 +72,7 @@ object ItemQueryGenerator { Condition.unit case _ => - Condition.Not(fromExpr(tables, coll)(inner)) + Condition.Not(fromExpr(today, tables, coll)(inner)) } case Expr.Exists(field) => @@ -87,9 +88,10 @@ object ItemQueryGenerator { } case Expr.SimpleExpr(op, Property.DateProperty(attr, value)) => - val dt = dateToTimestamp(value) - val col = timestampColumn(tables)(attr) - Condition.CompareVal(col, makeOp(op), dt) + val dt = dateToTimestamp(today)(value) + val col = timestampColumn(tables)(attr) + val noLikeOp = if (op == Operator.Like) Operator.Eq else op + Condition.CompareVal(col, makeOp(noLikeOp), dt) case Expr.InExpr(attr, values) => val col = stringColumn(tables)(attr) @@ -98,10 +100,18 @@ object ItemQueryGenerator { case Expr.InDateExpr(attr, values) => val col = timestampColumn(tables)(attr) - val dts = values.map(dateToTimestamp) + val dts = values.map(dateToTimestamp(today)) if (values.tail.isEmpty) col === dts.head else col.in(dts) + case Expr.DirectionExpr(incoming) => + if (incoming) tables.item.incoming === Direction.Incoming + else tables.item.incoming === Direction.Outgoing + + case Expr.InboxExpr(flag) => + if (flag) tables.item.state === ItemState.created + else tables.item.state === ItemState.confirmed + case Expr.TagIdsMatch(op, tags) => val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption) NonEmptyList @@ -142,12 +152,31 @@ object ItemQueryGenerator { Condition.unit } - private def dateToTimestamp(date: Date): Timestamp = + private def dateToTimestamp(today: LocalDate)(date: Date): Timestamp = date match { - case Date.Local(year, month, day) => - Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay()) + case d: Date.DateLiteral => + val ld = dateLiteralToDate(today)(d) + println(s">>>> date= $ld") + Timestamp.atUtc(ld.atStartOfDay) + case Date.Calc(date, c, period) => + val ld = c match { + case Date.CalcDirection.Plus => + dateLiteralToDate(today)(date).plus(period) + case Date.CalcDirection.Minus => + dateLiteralToDate(today)(date).minus(period) + } + println(s">>>> date= $ld") + Timestamp.atUtc(ld.atStartOfDay()) + } + + private def dateLiteralToDate(today: LocalDate)(dateLit: Date.DateLiteral): LocalDate = + dateLit match { + case Date.Local(date) => + date case Date.Millis(ms) => - Timestamp(Instant.ofEpochMilli(ms)) + Instant.ofEpochMilli(ms).atZone(Timestamp.UTC).toLocalDate() + case Date.Today => + today } private def anyColumn(tables: Tables)(attr: Attr): Column[_] = @@ -171,6 +200,7 @@ object ItemQueryGenerator { case Attr.ItemId => tables.item.id.cast[String] case Attr.ItemName => tables.item.name case Attr.ItemSource => tables.item.source + case Attr.ItemNotes => tables.item.notes case Attr.Correspondent.OrgId => tables.corrOrg.oid.cast[String] case Attr.Correspondent.OrgName => tables.corrOrg.name case Attr.Correspondent.PersonId => tables.corrPers.pid.cast[String] diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 7461cc8b..a79db262 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -1,5 +1,7 @@ package docspell.store.queries +import java.time.LocalDate + import cats.data.{NonEmptyList => Nel} import cats.effect.Sync import cats.effect.concurrent.Ref @@ -226,41 +228,42 @@ object QItem { findCustomFieldValuesForColl(coll, q.customValues) .map(itemIds => i.id.in(itemIds)) - def queryCondFromExpr(coll: Ident, q: ItemQuery): Condition = { + def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery): Condition = { val tables = Tables(i, org, pers0, pers1, equip, f, a, m) - ItemQueryGenerator.fromExpr(tables, coll)(q.expr) + ItemQueryGenerator.fromExpr(today, tables, coll)(q.expr) } - def queryCondition(coll: Ident, cond: Query.QueryCond): Condition = + def queryCondition(today: LocalDate, coll: Ident, cond: Query.QueryCond): Condition = cond match { case fm: Query.QueryForm => queryCondFromForm(coll, fm) case expr: Query.QueryExpr => - queryCondFromExpr(coll, expr.q) + queryCondFromExpr(today, coll, expr.q) } def findItems( q: Query, + today: LocalDate, maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { val sql = findItemsBase(q.fix, maxNoteLen) - .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .limit(batch) .build logger.trace(s"List $batch items: $sql") sql.query[ListItem].stream } - def searchStats(q: Query): ConnectionIO[SearchSummary] = + def searchStats(today: LocalDate)(q: Query): ConnectionIO[SearchSummary] = for { - count <- searchCountSummary(q) - tags <- searchTagSummary(q) - fields <- searchFieldSummary(q) - folders <- searchFolderSummary(q) + count <- searchCountSummary(today)(q) + tags <- searchTagSummary(today)(q) + fields <- searchFieldSummary(today)(q) + folders <- searchFolderSummary(today)(q) } yield SearchSummary(count, tags, fields, folders) - def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { + def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = from(ti) .innerJoin(tag, tag.tid === ti.tagId) @@ -270,7 +273,7 @@ object QItem { findItemsBase(q.fix, 0).unwrap .withSelect(select(tag.all).append(count(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) - .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(tag.tid) .build .query[TagCount] @@ -284,27 +287,27 @@ object QItem { } yield existing ++ other.map(TagCount(_, 0)) } - def searchCountSummary(q: Query): ConnectionIO[Int] = + def searchCountSummary(today: LocalDate)(q: Query): ConnectionIO[Int] = findItemsBase(q.fix, 0).unwrap .withSelect(Nel.of(count(i.id).as("num"))) - .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .build .query[Int] .unique - def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = { + def searchFolderSummary(today: LocalDate)(q: Query): ConnectionIO[List[FolderCount]] = { val fu = RUser.as("fu") findItemsBase(q.fix, 0).unwrap .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num"))) .changeFrom(_.innerJoin(fu, fu.uid === f.owner)) - .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(f.id, f.name, f.owner, fu.login) .build .query[FolderCount] .to[List] } - def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = { + def searchFieldSummary(today: LocalDate)(q: Query): ConnectionIO[List[FieldStats]] = { val fieldJoin = from(cv) .innerJoin(cf, cf.id === cv.field) @@ -313,7 +316,7 @@ object QItem { val base = findItemsBase(q.fix, 0).unwrap .changeFrom(_.prepend(fieldJoin)) - .changeWhere(c => c && queryCondition(q.fix.account.collective, q.cond)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(GroupBy(cf.all)) val basicFields = Nel.of( diff --git a/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala b/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala index 4bb5c57f..e2358f94 100644 --- a/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala +++ b/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala @@ -22,11 +22,12 @@ object ItemQueryGeneratorTest extends SimpleTestSuite { RAttachment.as("a"), RAttachmentMeta.as("m") ) + val now: LocalDate = LocalDate.of(2021, 2, 25) test("basic test") { val q = ItemQueryParser .parseUnsafe("(& name:hello date>=2020-02-01 (| source=expense folder=test ))") - val cond = ItemQueryGenerator(tables, Ident.unsafe("coll"))(q) + val cond = ItemQueryGenerator(now, tables, Ident.unsafe("coll"))(q) val expect = tables.item.name.like("hello") && tables.item.itemDate >= Timestamp.atUtc( LocalDate.of(2020, 2, 1).atStartOfDay() @@ -35,29 +36,4 @@ object ItemQueryGeneratorTest extends SimpleTestSuite { assertEquals(cond, expect) } -// test("migration2") { -// withStore("db2") { store => -// val c = RCollective( -// Ident.unsafe("coll1"), -// CollectiveState.Active, -// Language.German, -// true, -// Timestamp.Epoch -// ) -// val e = -// REquipment( -// Ident.unsafe("equip"), -// Ident.unsafe("coll1"), -// "name", -// Timestamp.Epoch, -// Timestamp.Epoch, -// None -// ) -// -// for { -// _ <- store.transact(RCollective.insert(c)) -// _ <- store.transact(REquipment.insert(e)).map(_ => ()) -// } yield () -// } -// } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index dac2bb0e..d737e1c9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -33,6 +33,7 @@ object Dependencies { val PoiVersion = "4.1.2" val PostgresVersion = "42.2.19" val PureConfigVersion = "0.14.1" + val ScalaJavaTimeVersion = "2.2.0" val Slf4jVersion = "1.7.30" val StanfordNlpVersion = "4.2.0" val TikaVersion = "1.25" @@ -54,6 +55,9 @@ object Dependencies { val catsJS = Def.setting("org.typelevel" %%% "cats-core" % "2.4.2") + val scalaJavaTime = + Def.setting("io.github.cquiroz" %%% "scala-java-time" % ScalaJavaTimeVersion) + val kittens = Seq( "org.typelevel" %% "kittens" % KittensVersion )