TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-21 18:08:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add more convenient date parsers and some basic macros

Browse Source
This commit is contained in:
Eike Kettner
2021-02-28 16:11:25 +01:00
parent af73b59ec2
commit 9013d9264e
23 changed files with 445 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
modules/query/src/main/scala/docspell/query/Date.scala
View File

@ -1,14 +1,32 @@
package docspell.query
sealed trait Date
object Date {
def apply(y: Int, m: Int, d: Int): Date =
Local(y, m, d)
import java.time.LocalDate
import java.time.Period
def apply(ms: Long): Date =
import cats.implicits._
sealed trait Date
object Date {
def apply(y: Int, m: Int, d: Int): Either[Throwable, DateLiteral] =
Either.catchNonFatal(Local(LocalDate.of(y, m, d)))
def apply(ms: Long): DateLiteral =
Millis(ms)
final case class Local(year: Int, month: Int, day: Int) extends Date
sealed trait DateLiteral extends Date
final case class Millis(ms: Long) extends Date
final case class Local(date: LocalDate) extends DateLiteral
final case class Millis(ms: Long) extends DateLiteral
case object Today extends DateLiteral
sealed trait CalcDirection
object CalcDirection {
case object Plus extends CalcDirection
case object Minus extends CalcDirection
}
case class Calc(date: DateLiteral, calc: CalcDirection, period: Period) extends Date
}

23
modules/query/src/main/scala/docspell/query/ItemQuery.scala
View File

@ -40,6 +40,7 @@ object ItemQuery {
case object ItemName extends StringAttr
case object ItemSource extends StringAttr
case object ItemNotes extends StringAttr
case object ItemId extends StringAttr
case object Date extends DateAttr
case object DueDate extends DateAttr
@ -69,6 +70,11 @@ object ItemQuery {
final case class StringProperty(attr: StringAttr, value: String) extends Property
final case class DateProperty(attr: DateAttr, value: Date) extends Property
def apply(sa: StringAttr, value: String): Property =
StringProperty(sa, value)
def apply(da: DateAttr, value: Date): Property =
DateProperty(da, value)
}
sealed trait Expr {
@ -88,6 +94,8 @@ object ItemQuery {
final case class Exists(field: Attr) extends Expr
final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr
final case class InboxExpr(inbox: Boolean) extends Expr
final case class DirectionExpr(incoming: Boolean) extends Expr
final case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr
final case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr
@ -97,6 +105,21 @@ object ItemQuery {
extends Expr
final case class Fulltext(query: String) extends Expr
def or(expr0: Expr, exprs: Expr*): OrExpr =
OrExpr(Nel.of(expr0, exprs: _*))
def and(expr0: Expr, exprs: Expr*): AndExpr =
AndExpr(Nel.of(expr0, exprs: _*))
def string(op: Operator, attr: StringAttr, value: String): SimpleExpr =
SimpleExpr(op, Property(attr, value))
def like(attr: StringAttr, value: String): SimpleExpr =
string(Operator.Like, attr, value)
def date(op: Operator, attr: DateAttr, value: Date): SimpleExpr =
SimpleExpr(op, Property(attr, value))
}
}

2
modules/query/src/main/scala/docspell/query/ItemQueryParser.scala
View File

@ -16,7 +16,7 @@ object ItemQueryParser {
ExprParser
.parseQuery(in)
.left
.map(pe => s"Error parsing: '$input': $pe")
.map(pe => s"Error parsing: '$input': $pe") //TODO
.map(q => q.copy(expr = ExprUtil.reduce(q.expr)))
}

34
modules/query/src/main/scala/docspell/query/internal/AttrParser.scala
View File

@ -7,57 +7,60 @@ import docspell.query.ItemQuery.Attr
object AttrParser {
val name: P[Attr.StringAttr] =
P.ignoreCase("name").map(_ => Attr.ItemName)
P.ignoreCase("name").as(Attr.ItemName)
val source: P[Attr.StringAttr] =
P.ignoreCase("source").map(_ => Attr.ItemSource)
P.ignoreCase("source").as(Attr.ItemSource)
val id: P[Attr.StringAttr] =
P.ignoreCase("id").map(_ => Attr.ItemId)
P.ignoreCase("id").as(Attr.ItemId)
val date: P[Attr.DateAttr] =
P.ignoreCase("date").map(_ => Attr.Date)
P.ignoreCase("date").as(Attr.Date)
val notes: P[Attr.StringAttr] =
P.ignoreCase("notes").as(Attr.ItemNotes)
val dueDate: P[Attr.DateAttr] =
P.stringIn(List("dueDate", "due", "due-date")).map(_ => Attr.DueDate)
P.stringIn(List("dueDate", "due", "due-date")).as(Attr.DueDate)
val corrOrgId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.id", "corr.org.id"))
.map(_ => Attr.Correspondent.OrgId)
.as(Attr.Correspondent.OrgId)
val corrOrgName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.name", "corr.org.name"))
.map(_ => Attr.Correspondent.OrgName)
.as(Attr.Correspondent.OrgName)
val corrPersId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.id", "corr.pers.id"))
.map(_ => Attr.Correspondent.PersonId)
.as(Attr.Correspondent.PersonId)
val corrPersName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.name", "corr.pers.name"))
.map(_ => Attr.Correspondent.PersonName)
.as(Attr.Correspondent.PersonName)
val concPersId: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.id", "conc.pers.id"))
.map(_ => Attr.Concerning.PersonId)
.as(Attr.Concerning.PersonId)
val concPersName: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.name", "conc.pers.name"))
.map(_ => Attr.Concerning.PersonName)
.as(Attr.Concerning.PersonName)
val concEquipId: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.id", "conc.equip.id"))
.map(_ => Attr.Concerning.EquipId)
.as(Attr.Concerning.EquipId)
val concEquipName: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.name", "conc.equip.name"))
.map(_ => Attr.Concerning.EquipName)
.as(Attr.Concerning.EquipName)
val folderId: P[Attr.StringAttr] =
P.ignoreCase("folder.id").map(_ => Attr.Folder.FolderId)
P.ignoreCase("folder.id").as(Attr.Folder.FolderId)
val folderName: P[Attr.StringAttr] =
P.ignoreCase("folder").map(_ => Attr.Folder.FolderName)
P.ignoreCase("folder").as(Attr.Folder.FolderName)
val dateAttr: P[Attr.DateAttr] =
P.oneOf(List(date, dueDate))
@ -68,6 +71,7 @@ object AttrParser {
name,
source,
id,
notes,
corrOrgId,
corrOrgName,
corrPersId,

6
modules/query/src/main/scala/docspell/query/internal/BasicParser.scala
View File

@ -38,4 +38,10 @@ object BasicParser {
val stringOrMore: P[Nel[String]] =
singleString.repSep(stringListSep)
val bool: P[Boolean] = {
val trueP = P.stringIn(List("yes", "true", "Yes", "True")).as(true)
val falseP = P.stringIn(List("no", "false", "No", "False")).as(false)
trueP | falseP
}
}

85
modules/query/src/main/scala/docspell/query/internal/DateParser.scala
View File

@ -1,7 +1,8 @@
package docspell.query.internal
import cats.data.NonEmptyList
import cats.implicits._
import java.time.Period
import cats.data.{NonEmptyList => Nel}
import cats.parse.{Numbers, Parser => P}
import docspell.query.Date
@ -26,21 +27,79 @@ object DateParser {
digits2.filter(n => n >= 1 && n <= 31)
private val dateSep: P[Unit] =
P.anyChar.void
P.charIn('-', '/').void
val localDateFromString: P[Date] =
((digits4 <* dateSep) ~ (month <* dateSep) ~ day).mapFilter {
case ((year, month), day) =>
Either.catchNonFatal(Date(year, month, day)).toOption
private val dateString: P[((Int, Option[Int]), Option[Int])] =
digits4 ~ (dateSep *> month).? ~ (dateSep *> day).?
private[internal] val dateFromString: P[Date.DateLiteral] =
dateString.mapFilter { case ((year, month), day) =>
Date(year, month.getOrElse(1), day.getOrElse(1)).toOption
}
val dateFromMillis: P[Date] =
longParser.map(Date.apply)
private[internal] val dateFromMillis: P[Date.DateLiteral] =
P.string("ms") *> longParser.map(Date.apply)
val localDate: P[Date] =
localDateFromString.backtrack.orElse(dateFromMillis)
private val dateFromToday: P[Date.DateLiteral] =
P.string("today").as(Date.Today)
val localDateOrMore: P[NonEmptyList[Date]] =
localDate.repSep(BasicParser.stringListSep)
val dateLiteral: P[Date.DateLiteral] =
P.oneOf(List(dateFromString, dateFromToday, dateFromMillis))
// val dateLiteralOrMore: P[NonEmptyList[Date.DateLiteral]] =
// dateLiteral.repSep(BasicParser.stringListSep)
val dateCalcDirection: P[Date.CalcDirection] =
P.oneOf(
List(
P.char('+').as(Date.CalcDirection.Plus),
P.char('-').as(Date.CalcDirection.Minus)
)
)
def periodPart(unitSuffix: Char, f: Int => Period): P[Period] =
((Numbers.nonZeroDigit ~ Numbers.digits0).void.string.soft <* P.ignoreCaseChar(
unitSuffix
))
.map(n => f(n.toInt))
private[this] val periodMonths: P[Period] =
periodPart('m', n => Period.ofMonths(n))
private[this] val periodDays: P[Period] =
periodPart('d', n => Period.ofDays(n))
val period: P[Period] =
periodDays.eitherOr(periodMonths).map(_.fold(identity, identity))
val periods: P[Period] =
period.rep.map(_.reduceLeft((p0, p1) => p0.plus(p1)))
val dateRange: P[(Date, Date)] =
((dateLiteral <* P.char(';')) ~ dateCalcDirection.eitherOr(P.char('/')) ~ period)
.map { case ((date, calc), period) =>
calc match {
case Right(Date.CalcDirection.Plus) =>
(date, Date.Calc(date, Date.CalcDirection.Plus, period))
case Right(Date.CalcDirection.Minus) =>
(Date.Calc(date, Date.CalcDirection.Minus, period), date)
case Left(_) =>
(
Date.Calc(date, Date.CalcDirection.Minus, period),
Date.Calc(date, Date.CalcDirection.Plus, period)
)
}
}
val date: P[Date] =
(dateLiteral ~ (P.char(';') *> dateCalcDirection ~ period).?).map {
case (date, Some((c, p))) =>
Date.Calc(date, c, p)
case (date, None) =>
date
}
val dateOrMore: P[Nel[Date]] =
date.repSep(BasicParser.stringListSep)
}

9
modules/query/src/main/scala/docspell/query/internal/ExprParser.scala
View File

@ -24,10 +24,11 @@ object ExprParser {
val exprParser: P[Expr] =
P.recursive[Expr] { recurse =>
val andP = and(recurse)
val orP = or(recurse)
val notP = not(recurse)
P.oneOf(SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil)
val andP = and(recurse)
val orP = or(recurse)
val notP = not(recurse)
val macros = MacroParser.all
P.oneOf(SimpleExprParser.simpleExpr :: macros :: andP :: orP :: notP :: Nil)
}
def parseQuery(input: String): Either[P.Error, ItemQuery] = {

10
modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala
View File

@ -22,10 +22,20 @@ object ExprUtil {
inner match {
case NotExpr(inner2) =>
reduce(inner2)
case InboxExpr(flag) =>
InboxExpr(!flag)
case DirectionExpr(flag) =>
DirectionExpr(!flag)
case _ =>
expr
}
case DirectionExpr(_) =>
expr
case InboxExpr(_) =>
expr
case InExpr(_, _) =>
expr

65
modules/query/src/main/scala/docspell/query/internal/MacroParser.scala Normal file
View File

@ -0,0 +1,65 @@
package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery._
object MacroParser {
private[this] val macroDef: P[String] =
P.char('$') *> BasicParser.identParser <* P.char(':')
def parser[A](macros: Map[String, P[A]]): P[A] = {
val p: P[P[A]] = macroDef.map { name =>
macros
.get(name)
.getOrElse(P.failWith(s"Unknown macro: $name"))
}
val px = (p ~ P.index ~ BasicParser.singleString).map { case ((pexpr, index), str) =>
pexpr
.parseAll(str)
.left
.map(err => err.copy(failedAtOffset = err.failedAtOffset + index))
}
P.select(px)(P.Fail)
}
// --- definitions of available macros
/** Expands in an OR expression that matches name fields of item and
* correspondent/concerning metadata.
*/
val names: P[Expr] =
P.string(P.anyChar.rep.void).map { input =>
Expr.or(
Expr.like(Attr.ItemName, input),
Expr.like(Attr.ItemNotes, input),
Expr.like(Attr.Correspondent.OrgName, input),
Expr.like(Attr.Correspondent.PersonName, input),
Expr.like(Attr.Concerning.PersonName, input),
Expr.like(Attr.Concerning.EquipName, input)
)
}
def dateRange(attr: Attr.DateAttr): P[Expr] =
DateParser.dateRange.map { case (left, right) =>
Expr.and(
Expr.date(Operator.Gte, attr, left),
Expr.date(Operator.Lte, attr, right)
)
}
// --- all macro parser
val allMacros: Map[String, P[Expr]] =
Map(
"names" -> names,
"datein" -> dateRange(Attr.Date),
"duein" -> dateRange(Attr.DueDate)
)
val all: P[Expr] =
parser(allMacros)
}

18
modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala
View File

@ -6,34 +6,34 @@ import docspell.query.ItemQuery._
object OperatorParser {
private[this] val Eq: P[Operator] =
P.char('=').void.map(_ => Operator.Eq)
P.char('=').as(Operator.Eq)
private[this] val Neq: P[Operator] =
P.string("!=").void.map(_ => Operator.Neq)
P.string("!=").as(Operator.Neq)
private[this] val Like: P[Operator] =
P.char(':').void.map(_ => Operator.Like)
P.char(':').as(Operator.Like)
private[this] val Gt: P[Operator] =
P.char('>').void.map(_ => Operator.Gt)
P.char('>').as(Operator.Gt)
private[this] val Lt: P[Operator] =
P.char('<').void.map(_ => Operator.Lt)
P.char('<').as(Operator.Lt)
private[this] val Gte: P[Operator] =
P.string(">=").map(_ => Operator.Gte)
P.string(">=").as(Operator.Gte)
private[this] val Lte: P[Operator] =
P.string("<=").map(_ => Operator.Lte)
P.string("<=").as(Operator.Lte)
val op: P[Operator] =
P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt))
private[this] val anyOp: P[TagOperator] =
P.char(':').map(_ => TagOperator.AnyMatch)
P.char(':').as(TagOperator.AnyMatch)
private[this] val allOp: P[TagOperator] =
P.char('=').map(_ => TagOperator.AllMatch)
P.char('=').as(TagOperator.AllMatch)
val tagOp: P[TagOperator] =
P.oneOf(List(anyOp, allOp))

12
modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala
View File

@ -17,7 +17,7 @@ object SimpleExprParser {
P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore)
private[this] val inOrOpDate =
P.eitherOr(op ~ DateParser.localDate, inOp *> DateParser.localDateOrMore)
P.eitherOr(op ~ DateParser.date, inOp *> DateParser.dateOrMore)
val stringExpr: P[Expr] =
(AttrParser.stringAttr ~ inOrOpStr).map {
@ -65,6 +65,12 @@ object SimpleExprParser {
CustomFieldMatch(name, op, value)
}
val inboxExpr: P[Expr.InboxExpr] =
(P.string("inbox:") *> BasicParser.bool).map(Expr.InboxExpr.apply)
val dirExpr: P[Expr.DirectionExpr] =
(P.string("incoming:") *> BasicParser.bool).map(Expr.DirectionExpr.apply)
val simpleExpr: P[Expr] =
P.oneOf(
List(
@ -75,7 +81,9 @@ object SimpleExprParser {
tagIdExpr,
tagExpr,
catExpr,
customFieldExpr
customFieldExpr,
inboxExpr,
dirExpr
)
)
}

51
modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala
View File

@ -2,35 +2,68 @@ package docspell.query.internal
import minitest._
import docspell.query.Date
import java.time.Period
object DateParserTest extends SimpleTestSuite {
def ld(year: Int, m: Int, d: Int): Date =
Date(year, m, d)
def ld(year: Int, m: Int, d: Int): Date.DateLiteral =
Date(year, m, d).fold(throw _, identity)
def ldPlus(year: Int, m: Int, d: Int, p: Period): Date.Calc =
Date.Calc(ld(year, m, d), Date.CalcDirection.Plus, p)
def ldMinus(year: Int, m: Int, d: Int, p: Period): Date.Calc =
Date.Calc(ld(year, m, d), Date.CalcDirection.Minus, p)
test("local date string") {
val p = DateParser.localDateFromString
val p = DateParser.dateFromString
assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22)))
assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11)))
assertEquals(p.parseAll("2032-01-21"), Right(ld(2032, 1, 21)))
assert(p.parseAll("0-0-0").isLeft)
assert(p.parseAll("2021-02-30").isRight)
assert(p.parseAll("2021-02-30").isLeft)
}
test("local date millis") {
val p = DateParser.dateFromMillis
assertEquals(p.parseAll("0"), Right(Date(0)))
assertEquals(p.parseAll("ms0"), Right(Date(0)))
assertEquals(
p.parseAll("1600000065463"),
p.parseAll("ms1600000065463"),
Right(Date(1600000065463L))
)
}
test("local date") {
val p = DateParser.localDate
val p = DateParser.date
assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22)))
assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11)))
assertEquals(p.parseAll("0"), Right(Date(0)))
assertEquals(p.parseAll("1600000065463"), Right(Date(1600000065463L)))
assertEquals(p.parseAll("ms0"), Right(Date(0)))
assertEquals(p.parseAll("ms1600000065463"), Right(Date(1600000065463L)))
}
test("local partial date") {
val p = DateParser.date
assertEquals(p.parseAll("2021-04"), Right(ld(2021, 4, 1)))
assertEquals(p.parseAll("2021-12"), Right(ld(2021, 12, 1)))
assert(p.parseAll("2021-13").isLeft)
assert(p.parseAll("2021-28").isLeft)
assertEquals(p.parseAll("2021"), Right(ld(2021, 1, 1)))
}
test("date calcs") {
val p = DateParser.date
assertEquals(p.parseAll("2020-02;+2d"), Right(ldPlus(2020, 2, 1, Period.ofDays(2))))
assertEquals(
p.parseAll("today;-2m"),
Right(Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2)))
)
}
test("period") {
val p = DateParser.periods
assertEquals(p.parseAll("15d"), Right(Period.ofDays(15)))
assertEquals(p.parseAll("15m"), Right(Period.ofMonths(15)))
assertEquals(p.parseAll("15d10m"), Right(Period.ofMonths(10).plus(Period.ofDays(15))))
assertEquals(p.parseAll("10m15d"), Right(Period.ofMonths(10).plus(Period.ofDays(15))))
}
}

19
modules/query/src/test/scala/docspell/query/internal/MacroParserTest.scala Normal file
View File

@ -0,0 +1,19 @@
package docspell.query.internal
import minitest._
import cats.parse.{Parser => P}
object MacroParserTest extends SimpleTestSuite {
test("fail with unkown macro names") {
val p = MacroParser.parser(Map.empty)
assert(p.parseAll("$bla:blup").isLeft) // TODO check error message
}
test("select correct parser") {
val p =
MacroParser.parser[Int](Map("one" -> P.anyChar.as(1), "two" -> P.anyChar.as(2)))
assertEquals(p.parseAll("$one:y"), Right(1))
assertEquals(p.parseAll("$two:y"), Right(2))
}
}

27
modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala
View File

@ -4,6 +4,7 @@ import cats.data.{NonEmptyList => Nel}
import docspell.query.ItemQuery._
import minitest._
import docspell.query.Date
import java.time.Period
object SimpleExprParserTest extends SimpleTestSuite {
@ -39,8 +40,8 @@ object SimpleExprParserTest extends SimpleTestSuite {
test("date expr") {
val p = SimpleExprParser.dateExpr
assertEquals(
p.parseAll("due:2021-03-14"),
Right(dateExpr(Operator.Like, Attr.DueDate, ld(2021, 3, 14)))
p.parseAll("date:2021-03-14"),
Right(dateExpr(Operator.Like, Attr.Date, ld(2021, 3, 14)))
)
assertEquals(
p.parseAll("due<2021-03-14"),
@ -50,6 +51,28 @@ object SimpleExprParserTest extends SimpleTestSuite {
p.parseAll("due~=2021-03-14,2021-03-13"),
Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13))))
)
assertEquals(
p.parseAll("due>2021"),
Right(dateExpr(Operator.Gt, Attr.DueDate, ld(2021, 1, 1)))
)
assertEquals(
p.parseAll("date<2021-01"),
Right(dateExpr(Operator.Lt, Attr.Date, ld(2021, 1, 1)))
)
assertEquals(
p.parseAll("date<today"),
Right(dateExpr(Operator.Lt, Attr.Date, Date.Today))
)
assertEquals(
p.parseAll("date>today;-2m"),
Right(
dateExpr(
Operator.Gt,
Attr.Date,
Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2))
)
)
)
}
test("exists expr") {