TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-10-31 17:50:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parser improvements

Browse Source 
- default expressions into a and node
- fix parsing string lists that end in whitespace
- fix package names of internal classes
This commit is contained in:
Eike Kettner
2021-02-27 18:06:59 +01:00
parent a80d73d5d2
commit af73b59ec2
18 changed files with 229 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.sbt
View File

@@ -269,6 +269,7 @@ val query =
crossProject(JSPlatform, JVMPlatform)
.crossType(CrossType.Pure)
.in(file("modules/query"))
.disablePlugins(RevolverPlugin)
.settings(sharedSettings)
.settings(testSettings)
.settings(
@@ -596,6 +597,7 @@ val website = project
val root = project
.in(file("."))
.disablePlugins(RevolverPlugin)
.settings(sharedSettings)
.settings(noPublish)
.settings(

26
modules/query/src/main/scala/docspell/query/ItemQuery.scala
View File

@@ -14,10 +14,12 @@ import docspell.query.ItemQuery.Attr.{DateAttr, StringAttr}
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String])
object ItemQuery {
val all = ItemQuery(Expr.Exists(Attr.ItemId), Some(""))
sealed trait Operator
object Operator {
case object Eq extends Operator
case object Neq extends Operator
case object Like extends Operator
case object Gt extends Operator
case object Lt extends Operator
@@ -75,24 +77,26 @@ object ItemQuery {
}
object Expr {
case class AndExpr(expr: Nel[Expr]) extends Expr
case class OrExpr(expr: Nel[Expr]) extends Expr
case class NotExpr(expr: Expr) extends Expr {
final case class AndExpr(expr: Nel[Expr]) extends Expr
final case class OrExpr(expr: Nel[Expr]) extends Expr
final case class NotExpr(expr: Expr) extends Expr {
override def negate: Expr =
expr
}
case class SimpleExpr(op: Operator, prop: Property) extends Expr
case class Exists(field: Attr) extends Expr
case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
final case class SimpleExpr(op: Operator, prop: Property) extends Expr
final case class Exists(field: Attr) extends Expr
final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr
case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr
case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr
case class TagCategoryMatch(op: TagOperator, cats: Nel[String]) extends Expr
final case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr
final case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr
final case class TagCategoryMatch(op: TagOperator, cats: Nel[String]) extends Expr
case class CustomFieldMatch(name: String, op: Operator, value: String) extends Expr
final case class CustomFieldMatch(name: String, op: Operator, value: String)
extends Expr
case class Fulltext(query: String) extends Expr
final case class Fulltext(query: String) extends Expr
}
}

15
modules/query/src/main/scala/docspell/query/ItemQueryParser.scala
View File

@@ -3,17 +3,22 @@ package docspell.query
import scala.scalajs.js.annotation._
import docspell.query.internal.ExprParser
import docspell.query.internal.ExprUtil
@JSExportTopLevel("DsItemQueryParser")
object ItemQueryParser {
@JSExport
def parse(input: String): Either[String, ItemQuery] =
ExprParser.exprParser
.parseAll(input.trim)
.left
.map(pe => s"Error parsing: '${input.trim}': $pe")
.map(expr => ItemQuery(expr, Some(input.trim)))
if (input.isEmpty) Right(ItemQuery.all)
else {
val in = if (input.charAt(0) == '(') input else s"(& $input )"
ExprParser
.parseQuery(in)
.left
.map(pe => s"Error parsing: '$input': $pe")
.map(q => q.copy(expr = ExprUtil.reduce(q.expr)))
}
def parseUnsafe(input: String): ItemQuery =
parse(input).fold(sys.error, identity)

20
modules/query/src/main/scala/docspell/query/internal/BasicParser.scala
View File

@@ -7,17 +7,14 @@ object BasicParser {
private[this] val whitespace: P[Unit] = P.charIn(" \t\r\n").void
val ws0: Parser0[Unit] = whitespace.rep0.void
val ws1: P[Unit] = whitespace.rep(1).void
val ws1: P[Unit] = whitespace.rep.void
private[this] val listSep: P[Unit] =
P.char(',').surroundedBy(BasicParser.ws0).void
def rep[A](pa: P[A]): P[Nel[A]] =
pa.repSep(listSep)
val stringListSep: P[Unit] =
(ws0.with1.soft ~ P.char(',') ~ ws0).void
private[this] val basicString: P[String] =
P.charsWhile(c =>
c > ' ' && c != '"' && c != '\\' && c != ',' && c != '[' && c != ']'
c > ' ' && c != '"' && c != '\\' && c != ',' && c != '[' && c != ']' && c != '(' && c != ')'
)
private[this] val identChars: Set[Char] =
@@ -38,14 +35,7 @@ object BasicParser {
val singleString: P[String] =
basicString.backtrack.orElse(StringUtil.quoted('"'))
val stringListValue: P[Nel[String]] = rep(singleString).with1
.between(P.char('['), P.char(']'))
.backtrack
.orElse(rep(singleString))
val stringOrMore: P[Nel[String]] =
stringListValue.backtrack.orElse(
singleString.map(v => Nel.of(v))
)
singleString.repSep(stringListSep)
}

4
modules/query/src/main/scala/docspell/query/internal/DateParser.scala
View File

@@ -1,5 +1,6 @@
package docspell.query.internal
import cats.data.NonEmptyList
import cats.implicits._
import cats.parse.{Numbers, Parser => P}
@@ -39,4 +40,7 @@ object DateParser {
val localDate: P[Date] =
localDateFromString.backtrack.orElse(dateFromMillis)
val localDateOrMore: P[NonEmptyList[Date]] =
localDate.repSep(BasicParser.stringListSep)
}

10
modules/query/src/main/scala/docspell/query/internal/ExprParser.scala
View File

@@ -2,6 +2,7 @@ package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery
import docspell.query.ItemQuery._
object ExprParser {
@@ -18,8 +19,8 @@ object ExprParser {
.between(BasicParser.parenOr, BasicParser.parenClose)
.map(Expr.OrExpr.apply)
def not(inner: P[Expr]): P[Expr.NotExpr] =
(P.char('!') *> inner).map(Expr.NotExpr.apply)
def not(inner: P[Expr]): P[Expr] =
(P.char('!') *> inner).map(_.negate)
val exprParser: P[Expr] =
P.recursive[Expr] { recurse =>
@@ -28,4 +29,9 @@ object ExprParser {
val notP = not(recurse)
P.oneOf(SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil)
}
def parseQuery(input: String): Either[P.Error, ItemQuery] = {
val p = BasicParser.ws0 *> exprParser <* (BasicParser.ws0 ~ P.end)
p.parseAll(input).map(expr => ItemQuery(expr, Some(input)))
}
}

50
modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala Normal file
View File

@@ -0,0 +1,50 @@
package docspell.query.internal
import docspell.query.ItemQuery.Expr._
import docspell.query.ItemQuery._
object ExprUtil {
/** Does some basic transformation, like unfolding deeply nested and
* trees containing one value etc.
*/
def reduce(expr: Expr): Expr =
expr match {
case AndExpr(inner) =>
if (inner.tail.isEmpty) reduce(inner.head)
else AndExpr(inner.map(reduce))
case OrExpr(inner) =>
if (inner.tail.isEmpty) reduce(inner.head)
else OrExpr(inner.map(reduce))
case NotExpr(inner) =>
inner match {
case NotExpr(inner2) =>
reduce(inner2)
case _ =>
expr
}
case InExpr(_, _) =>
expr
case InDateExpr(_, _) =>
expr
case TagsMatch(_, _) =>
expr
case TagIdsMatch(_, _) =>
expr
case Exists(_) =>
expr
case Fulltext(_) =>
expr
case SimpleExpr(_, _) =>
expr
case TagCategoryMatch(_, _) =>
expr
case CustomFieldMatch(_, _, _) =>
expr
}
}

5
modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala
View File

@@ -8,6 +8,9 @@ object OperatorParser {
private[this] val Eq: P[Operator] =
P.char('=').void.map(_ => Operator.Eq)
private[this] val Neq: P[Operator] =
P.string("!=").void.map(_ => Operator.Neq)
private[this] val Like: P[Operator] =
P.char(':').void.map(_ => Operator.Like)
@@ -24,7 +27,7 @@ object OperatorParser {
P.string("<=").map(_ => Operator.Lte)
val op: P[Operator] =
P.oneOf(List(Like, Eq, Gte, Lte, Gt, Lt))
P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt))
private[this] val anyOp: P[TagOperator] =
P.char(':').map(_ => TagOperator.AnyMatch)

26
modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala
View File

@@ -10,15 +10,29 @@ object SimpleExprParser {
private[this] val op: P[Operator] =
OperatorParser.op.surroundedBy(BasicParser.ws0)
val stringExpr: P[Expr.SimpleExpr] =
(AttrParser.stringAttr ~ op ~ BasicParser.singleString).map {
case ((attr, op), value) =>
private[this] val inOp: P[Unit] =
P.string("~=").surroundedBy(BasicParser.ws0)
private[this] val inOrOpStr =
P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore)
private[this] val inOrOpDate =
P.eitherOr(op ~ DateParser.localDate, inOp *> DateParser.localDateOrMore)
val stringExpr: P[Expr] =
(AttrParser.stringAttr ~ inOrOpStr).map {
case (attr, Right((op, value))) =>
Expr.SimpleExpr(op, Property.StringProperty(attr, value))
case (attr, Left(values)) =>
Expr.InExpr(attr, values)
}
val dateExpr: P[Expr.SimpleExpr] =
(AttrParser.dateAttr ~ op ~ DateParser.localDate).map { case ((attr, op), value) =>
Expr.SimpleExpr(op, Property.DateProperty(attr, value))
val dateExpr: P[Expr] =
(AttrParser.dateAttr ~ inOrOpDate).map {
case (attr, Right((op, value))) =>
Expr.SimpleExpr(op, Property.DateProperty(attr, value))
case (attr, Left(values)) =>
Expr.InDateExpr(attr, values)
}
val existsExpr: P[Expr.Exists] =

2
modules/query/src/test/scala/docspell/query/AttrParserTest.scala → modules/query/src/test/scala/docspell/query/internal/AttrParserTest.scala
View File

@@ -1,4 +1,4 @@
package docspell.query
package docspell.query.internal
import docspell.query.ItemQuery.Attr
import docspell.query.internal.AttrParser

13
modules/query/src/test/scala/docspell/query/BasicParserTest.scala → modules/query/src/test/scala/docspell/query/internal/BasicParserTest.scala
View File

@@ -1,4 +1,4 @@
package docspell.query
package docspell.query.internal
import minitest._
import cats.data.{NonEmptyList => Nel}
@@ -14,13 +14,7 @@ object BasicParserTest extends SimpleTestSuite {
}
test("string list values") {
val p = BasicParser.stringListValue
assertEquals(p.parseAll("[ab,cd]"), Right(Nel.of("ab", "cd")))
assertEquals(p.parseAll("[\"ab 12\",cd]"), Right(Nel.of("ab 12", "cd")))
assertEquals(
p.parseAll("[\"ab, 12\",cd]"),
Right(Nel.of("ab, 12", "cd"))
)
val p = BasicParser.stringOrMore
assertEquals(p.parseAll("ab,cd,123"), Right(Nel.of("ab", "cd", "123")))
assertEquals(p.parseAll("a,b"), Right(Nel.of("a", "b")))
assert(p.parseAll("[a,b").isLeft)
@@ -30,6 +24,7 @@ object BasicParserTest extends SimpleTestSuite {
val p = BasicParser.stringOrMore
assertEquals(p.parseAll("abcde"), Right(Nel.of("abcde")))
assertEquals(p.parseAll(""""a,b,c""""), Right(Nel.of("a,b,c")))
assertEquals(p.parseAll("[a,b,c]"), Right(Nel.of("a", "b", "c")))
assertEquals(p.parse("a, b, c "), Right((" ", Nel.of("a", "b", "c"))))
}
}

4
modules/query/src/test/scala/docspell/query/DateParserTest.scala → modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala
View File

@@ -1,7 +1,7 @@
package docspell.query
package docspell.query.internal
import docspell.query.internal.DateParser
import minitest._
import docspell.query.Date
object DateParserTest extends SimpleTestSuite {

29
modules/query/src/test/scala/docspell/query/ExprParserTest.scala → modules/query/src/test/scala/docspell/query/internal/ExprParserTest.scala
View File

@@ -1,8 +1,7 @@
package docspell.query
package docspell.query.internal
import docspell.query.ItemQuery._
import docspell.query.SimpleExprParserTest.stringExpr
import docspell.query.internal.ExprParser
import docspell.query.internal.SimpleExprParserTest.stringExpr
import minitest._
import cats.data.{NonEmptyList => Nel}
@@ -45,4 +44,28 @@ object ExprParserTest extends SimpleTestSuite {
)
)
}
test("tag list inside and/or") {
val p = ExprParser.exprParser
assertEquals(
p.parseAll("(& tag:a,b,c)"),
Right(
Expr.AndExpr(
Nel.of(
Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c"))
)
)
)
)
assertEquals(
p.parseAll("(& tag:a,b,c )"),
Right(
Expr.AndExpr(
Nel.of(
Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c"))
)
)
)
)
}
}

43
modules/query/src/test/scala/docspell/query/internal/ItemQueryParserTest.scala Normal file
View File

@@ -0,0 +1,43 @@
package docspell.query.internal
import minitest._
import docspell.query.ItemQueryParser
import docspell.query.ItemQuery
object ItemQueryParserTest extends SimpleTestSuite {
test("reduce ands") {
val q = ItemQueryParser.parseUnsafe("(&(&(&(& name:hello))))")
val expr = ExprUtil.reduce(q.expr)
assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
}
test("reduce ors") {
val q = ItemQueryParser.parseUnsafe("(|(|(|(| name:hello))))")
val expr = ExprUtil.reduce(q.expr)
assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
}
test("reduce and/or") {
val q = ItemQueryParser.parseUnsafe("(|(&(&(| name:hello))))")
val expr = ExprUtil.reduce(q.expr)
assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
}
test("reduce inner and/or") {
val q = ItemQueryParser.parseUnsafe("(& name:hello (| name:world))")
val expr = ExprUtil.reduce(q.expr)
assertEquals(expr, ItemQueryParser.parseUnsafe("(& name:hello name:world)").expr)
}
test("omit and-parens around root structure") {
val q = ItemQueryParser.parseUnsafe("name:hello date>2020-02-02")
val expect = ItemQueryParser.parseUnsafe("(& name:hello date>2020-02-02 )")
assertEquals(expect, q)
}
test("return all if query is empty") {
val q = ItemQueryParser.parseUnsafe("")
assertEquals(ItemQuery.all, q)
}
}

3
modules/query/src/test/scala/docspell/query/OperatorParserTest.scala → modules/query/src/test/scala/docspell/query/internal/OperatorParserTest.scala
View File

@@ -1,4 +1,4 @@
package docspell.query
package docspell.query.internal
import minitest._
import docspell.query.ItemQuery.{Operator, TagOperator}
@@ -8,6 +8,7 @@ object OperatorParserTest extends SimpleTestSuite {
test("operator values") {
val p = OperatorParser.op
assertEquals(p.parseAll("="), Right(Operator.Eq))
assertEquals(p.parseAll("!="), Right(Operator.Neq))
assertEquals(p.parseAll(":"), Right(Operator.Like))
assertEquals(p.parseAll("<"), Right(Operator.Lt))
assertEquals(p.parseAll(">"), Right(Operator.Gt))

13
modules/query/src/test/scala/docspell/query/SimpleExprParserTest.scala → modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala
View File

@@ -1,9 +1,9 @@
package docspell.query
package docspell.query.internal
import cats.data.{NonEmptyList => Nel}
import docspell.query.ItemQuery._
import docspell.query.internal.SimpleExprParser
import minitest._
import docspell.query.Date
object SimpleExprParserTest extends SimpleTestSuite {
@@ -29,6 +29,11 @@ object SimpleExprParserTest extends SimpleTestSuite {
p.parseAll("conc.pers.id=Aaiet-aied"),
Right(stringExpr(Operator.Eq, Attr.Concerning.PersonId, "Aaiet-aied"))
)
assert(p.parseAll("conc.pers.id=Aaiet,aied").isLeft)
assertEquals(
p.parseAll("name~=hello,world"),
Right(Expr.InExpr(Attr.ItemName, Nel.of("hello", "world")))
)
}
test("date expr") {
@@ -41,6 +46,10 @@ object SimpleExprParserTest extends SimpleTestSuite {
p.parseAll("due<2021-03-14"),
Right(dateExpr(Operator.Lt, Attr.DueDate, ld(2021, 3, 14)))
)
assertEquals(
p.parseAll("due~=2021-03-14,2021-03-13"),
Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13))))
)
}
test("exists expr") {

8
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@@ -51,13 +51,11 @@ object ItemRoutes {
offset
) =>
val query =
q.map(ItemQueryParser.parse) match {
case Some(Right(q)) =>
ItemQueryParser.parse(q.getOrElse("")) match {
case Right(q) =>
Right(Query(Query.Fix(user.account, None, None), Query.QueryExpr(q)))
case Some(Left(err)) =>
case Left(err) =>
Left(err)
case None =>
Right(Query(Query.Fix(user.account, None, None), Query.QueryForm.empty))
}
val li = limit.getOrElse(cfg.maxItemPageSize)
val of = offset.getOrElse(0)

26
modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala
View File

@@ -12,6 +12,7 @@ import docspell.store.qb.{Operator => QOp, _}
import docspell.store.records.{RCustomField, RCustomFieldValue, TagItemName}
import doobie.util.Put
import docspell.store.queries.QueryWildcard
object ItemQueryGenerator {
@@ -80,18 +81,13 @@ object ItemQueryGenerator {
val col = stringColumn(tables)(attr)
op match {
case Operator.Like =>
Condition.CompareVal(col, makeOp(op), value.toLowerCase)
Condition.CompareVal(col, makeOp(op), QueryWildcard.lower(value))
case _ =>
Condition.CompareVal(col, makeOp(op), value)
}
case Expr.SimpleExpr(op, Property.DateProperty(attr, value)) =>
val dt = value match {
case Date.Local(year, month, day) =>
Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay())
case Date.Millis(ms) =>
Timestamp(Instant.ofEpochMilli(ms))
}
val dt = dateToTimestamp(value)
val col = timestampColumn(tables)(attr)
Condition.CompareVal(col, makeOp(op), dt)
@@ -100,6 +96,12 @@ object ItemQueryGenerator {
if (values.tail.isEmpty) col === values.head
else col.in(values)
case Expr.InDateExpr(attr, values) =>
val col = timestampColumn(tables)(attr)
val dts = values.map(dateToTimestamp)
if (values.tail.isEmpty) col === dts.head
else col.in(dts)
case Expr.TagIdsMatch(op, tags) =>
val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption)
NonEmptyList
@@ -140,6 +142,14 @@ object ItemQueryGenerator {
Condition.unit
}
private def dateToTimestamp(date: Date): Timestamp =
date match {
case Date.Local(year, month, day) =>
Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay())
case Date.Millis(ms) =>
Timestamp(Instant.ofEpochMilli(ms))
}
private def anyColumn(tables: Tables)(attr: Attr): Column[_] =
attr match {
case s: Attr.StringAttr =>
@@ -177,6 +187,8 @@ object ItemQueryGenerator {
operator match {
case Operator.Eq =>
QOp.Eq
case Operator.Neq =>
QOp.Neq
case Operator.Like =>
QOp.LowerLike
case Operator.Gt =>