TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-10-31 09:30:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring parser

Browse Source 
- put all used strings in one place to have it easier to track
- don't use `$` for shortcuts, it's a detail not interesting to a
  user; now names must not clash (which is a good idea anyways)
- Added two more shortcuts `conc` and `corr`
This commit is contained in:
Eike Kettner
2021-03-08 22:46:40 +01:00
parent e681ffa96f
commit 77a87782b7
14 changed files with 183 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala
View File

@@ -123,23 +123,40 @@ object ItemQuery {
sealed trait MacroExpr extends Expr {
def body: Expr
}
case class NamesMacro(searchTerm: String) extends MacroExpr {
final case class NamesMacro(searchTerm: String) extends MacroExpr {
val body =
Expr.or(
like(Attr.ItemName, searchTerm),
like(Attr.ItemNotes, searchTerm),
like(Attr.Correspondent.OrgName, searchTerm),
like(Attr.Correspondent.PersonName, searchTerm),
like(Attr.Concerning.PersonName, searchTerm),
like(Attr.Concerning.EquipName, searchTerm)
)
}
case class DateRangeMacro(attr: DateAttr, left: Date, right: Date) extends MacroExpr {
final case class CorrMacro(term: String) extends MacroExpr {
val body =
and(date(Operator.Gte, attr, left), date(Operator.Lte, attr, right))
Expr.or(
like(Attr.Correspondent.OrgName, term),
like(Attr.Correspondent.PersonName, term)
)
}
case class YearMacro(attr: DateAttr, year: Int) extends MacroExpr {
final case class ConcMacro(term: String) extends MacroExpr {
val body =
Expr.or(
like(Attr.Concerning.PersonName, term),
like(Attr.Concerning.EquipName, term)
)
}
final case class DateRangeMacro(attr: DateAttr, left: Date, right: Date)
extends MacroExpr {
val body =
and(date(Operator.Gte, attr, left), date(Operator.Lt, attr, right))
}
final case class YearMacro(attr: DateAttr, year: Int) extends MacroExpr {
val body =
DateRangeMacro(attr, date(year), date(year + 1))

21
modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala
View File

@@ -31,9 +31,10 @@ object ParseFailure {
}
final case class SimpleMessage(offset: Int, msg: String) extends Message {
def render: String =
s"Failed at $offset: $msg"
s"Failed at $offset: $msg"
}
final case class ExpectMessage(offset: Int, expected: List[String], exhaustive: Boolean) extends Message {
final case class ExpectMessage(offset: Int, expected: List[String], exhaustive: Boolean)
extends Message {
def render: String = {
val opts = expected.mkString(", ")
val dots = if (exhaustive) "" else "…"
@@ -50,7 +51,8 @@ object ParseFailure {
private[query] def packMsg(msg: Nel[Message]): Nel[Message] = {
val expectMsg = combineExpected(msg.collect({ case em: ExpectMessage => em }))
.sortBy(_.offset).headOption
.sortBy(_.offset)
.headOption
val simpleMsg = msg.collect({ case sm: SimpleMessage => sm })
@@ -58,9 +60,16 @@ object ParseFailure {
}
private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] =
msg.groupBy(_.offset).map({ case (offset, es) =>
ExpectMessage(offset, es.flatMap(_.expected).distinct.sorted, es.forall(_.exhaustive))
}).toList
msg
.groupBy(_.offset)
.map({ case (offset, es) =>
ExpectMessage(
offset,
es.flatMap(_.expected).distinct.sorted,
es.forall(_.exhaustive)
)
})
.toList
private[query] def expectationToMsg(e: Parser.Expectation): Message =
e match {

35
modules/query/shared/src/main/scala/docspell/query/internal/AttrParser.scala
View File

@@ -3,67 +3,68 @@ package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery.Attr
import docspell.query.internal.{Constants => C}
object AttrParser {
val name: P[Attr.StringAttr] =
P.ignoreCase("name").as(Attr.ItemName)
P.ignoreCase(C.name).as(Attr.ItemName)
val source: P[Attr.StringAttr] =
P.ignoreCase("source").as(Attr.ItemSource)
P.ignoreCase(C.source).as(Attr.ItemSource)
val id: P[Attr.StringAttr] =
P.ignoreCase("id").as(Attr.ItemId)
P.ignoreCase(C.id).as(Attr.ItemId)
val date: P[Attr.DateAttr] =
P.ignoreCase("date").as(Attr.Date)
P.ignoreCase(C.date).as(Attr.Date)
val notes: P[Attr.StringAttr] =
P.ignoreCase("notes").as(Attr.ItemNotes)
P.ignoreCase(C.notes).as(Attr.ItemNotes)
val dueDate: P[Attr.DateAttr] =
P.stringIn(List("dueDate", "due", "due-date")).as(Attr.DueDate)
P.ignoreCase(C.due).as(Attr.DueDate)
val corrOrgId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.id", "corr.org.id"))
P.ignoreCase(C.corrOrgId)
.as(Attr.Correspondent.OrgId)
val corrOrgName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.org.name", "corr.org.name"))
P.ignoreCase(C.corrOrgName)
.as(Attr.Correspondent.OrgName)
val corrPersId: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.id", "corr.pers.id"))
P.ignoreCase(C.corrPersId)
.as(Attr.Correspondent.PersonId)
val corrPersName: P[Attr.StringAttr] =
P.stringIn(List("correspondent.person.name", "corr.pers.name"))
P.ignoreCase(C.corrPersName)
.as(Attr.Correspondent.PersonName)
val concPersId: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.id", "conc.pers.id"))
P.ignoreCase(C.concPersId)
.as(Attr.Concerning.PersonId)
val concPersName: P[Attr.StringAttr] =
P.stringIn(List("concerning.person.name", "conc.pers.name"))
P.ignoreCase(C.concPersName)
.as(Attr.Concerning.PersonName)
val concEquipId: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.id", "conc.equip.id"))
P.ignoreCase(C.concEquipId)
.as(Attr.Concerning.EquipId)
val concEquipName: P[Attr.StringAttr] =
P.stringIn(List("concerning.equip.name", "conc.equip.name"))
P.ignoreCase(C.concEquipName)
.as(Attr.Concerning.EquipName)
val folderId: P[Attr.StringAttr] =
P.ignoreCase("folder.id").as(Attr.Folder.FolderId)
P.ignoreCase(C.folderId).as(Attr.Folder.FolderId)
val folderName: P[Attr.StringAttr] =
P.ignoreCase("folder").as(Attr.Folder.FolderName)
P.ignoreCase(C.folder).as(Attr.Folder.FolderName)
val attachCountAttr: P[Attr.IntAttr] =
P.ignoreCase("attach.count").as(Attr.AttachCount)
P.ignoreCase(C.attachCount).as(Attr.AttachCount)
// combining grouped by type

50
modules/query/shared/src/main/scala/docspell/query/internal/Constants.scala Normal file
View File

@@ -0,0 +1,50 @@
package docspell.query.internal
object Constants {
val attachCount = "attach.count"
val attachId = "attach.id"
val cat = "cat"
val checksum = "checksum"
val conc = "conc"
val concEquipId = "conc.equip.id"
val concEquipName = "conc.equip.name"
val concPersId = "conc.pers.id"
val concPersName = "conc.pers.name"
val content = "content"
val corr = "corr"
val corrOrgId = "corr.org.id"
val corrOrgName = "corr.org.name"
val corrPersId = "corr.pers.id"
val corrPersName = "corr.pers.name"
val customField = "f"
val customFieldId = "f.id"
val date = "date"
val dateIn = "dateIn"
val due = "due"
val dueIn = "dueIn"
val exist = "exist"
val folder = "folder"
val folderId = "folder.id"
val id = "id"
val inbox = "inbox"
val incoming = "incoming"
val name = "name"
val names = "names"
val notPrefix = '!'
val notes = "notes"
val source = "source"
val tag = "tag"
val tagId = "tag.id"
val year = "year"
// operators
val eqs = '='
val gt = '>'
val gte = ">="
val in = "~="
val like = ':'
val lt = '<'
val lte = "<="
val neq = "!="
}

5
modules/query/shared/src/main/scala/docspell/query/internal/ExprParser.scala
View File

@@ -4,6 +4,7 @@ import cats.parse.{Parser => P}
import docspell.query.ItemQuery
import docspell.query.ItemQuery._
import docspell.query.internal.{Constants => C}
object ExprParser {
@@ -20,7 +21,7 @@ object ExprParser {
.map(Expr.OrExpr.apply)
def not(inner: P[Expr]): P[Expr] =
(P.char('!') *> inner).map(_.negate)
(P.char(C.notPrefix) *> inner).map(_.negate)
val exprParser: P[Expr] =
P.recursive[Expr] { recurse =>
@@ -28,7 +29,7 @@ object ExprParser {
val orP = or(recurse)
val notP = not(recurse)
val macros = MacroParser.all
P.oneOf(SimpleExprParser.simpleExpr :: macros :: andP :: orP :: notP :: Nil)
P.oneOf(macros :: SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil)
}
def parseQuery(input: String): Either[P.Error, ItemQuery] = {

28
modules/query/shared/src/main/scala/docspell/query/internal/MacroParser.scala
View File

@@ -3,10 +3,11 @@ package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery._
import docspell.query.internal.{Constants => C}
object MacroParser {
private def macroDef(name: String): P[Unit] =
P.char('$').soft.with1 *> P.string(name) <* P.char(':')
P.ignoreCase(name).soft.with1 <* P.char(':')
private def dateRangeMacroImpl(
name: String,
@@ -20,20 +21,35 @@ object MacroParser {
(macroDef(name) *> DateParser.yearOnly).map(year => Expr.YearMacro(attr, year))
val namesMacro: P[Expr.NamesMacro] =
(macroDef("names") *> BasicParser.singleString).map(Expr.NamesMacro.apply)
(macroDef(C.names) *> BasicParser.singleString).map(Expr.NamesMacro.apply)
val dateRangeMacro: P[Expr.DateRangeMacro] =
dateRangeMacroImpl("datein", Attr.Date)
dateRangeMacroImpl(C.dateIn, Attr.Date)
val dueDateRangeMacro: P[Expr.DateRangeMacro] =
dateRangeMacroImpl("duein", Attr.DueDate)
dateRangeMacroImpl(C.dueIn, Attr.DueDate)
val yearDateMacro: P[Expr.YearMacro] =
yearMacroImpl("year", Attr.Date)
yearMacroImpl(C.year, Attr.Date)
val corrMacro: P[Expr.CorrMacro] =
(macroDef(C.corr) *> BasicParser.singleString).map(Expr.CorrMacro.apply)
val concMacro: P[Expr.ConcMacro] =
(macroDef(C.conc) *> BasicParser.singleString).map(Expr.ConcMacro.apply)
// --- all macro parser
val all: P[Expr] =
P.oneOf(List(namesMacro, dateRangeMacro, dueDateRangeMacro, yearDateMacro))
P.oneOf(
List(
namesMacro,
dateRangeMacro,
dueDateRangeMacro,
yearDateMacro,
corrMacro,
concMacro
)
)
}

19
modules/query/shared/src/main/scala/docspell/query/internal/OperatorParser.scala
View File

@@ -3,37 +3,38 @@ package docspell.query.internal
import cats.parse.{Parser => P}
import docspell.query.ItemQuery._
import docspell.query.internal.{Constants => C}
object OperatorParser {
private[this] val Eq: P[Operator] =
P.char('=').as(Operator.Eq)
P.char(C.eqs).as(Operator.Eq)
private[this] val Neq: P[Operator] =
P.string("!=").as(Operator.Neq)
P.string(C.neq).as(Operator.Neq)
private[this] val Like: P[Operator] =
P.char(':').as(Operator.Like)
P.char(C.like).as(Operator.Like)
private[this] val Gt: P[Operator] =
P.char('>').as(Operator.Gt)
P.char(C.gt).as(Operator.Gt)
private[this] val Lt: P[Operator] =
P.char('<').as(Operator.Lt)
P.char(C.lt).as(Operator.Lt)
private[this] val Gte: P[Operator] =
P.string(">=").as(Operator.Gte)
P.string(C.gte).as(Operator.Gte)
private[this] val Lte: P[Operator] =
P.string("<=").as(Operator.Lte)
P.string(C.lte).as(Operator.Lte)
val op: P[Operator] =
P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt))
private[this] val anyOp: P[TagOperator] =
P.char(':').as(TagOperator.AnyMatch)
P.char(C.like).as(TagOperator.AnyMatch)
private[this] val allOp: P[TagOperator] =
P.char('=').as(TagOperator.AllMatch)
P.char(C.eqs).as(TagOperator.AllMatch)
val tagOp: P[TagOperator] =
P.oneOf(List(anyOp, allOp))

44
modules/query/shared/src/main/scala/docspell/query/internal/SimpleExprParser.scala
View File

@@ -4,6 +4,7 @@ import cats.parse.Numbers
import cats.parse.{Parser => P}
import docspell.query.ItemQuery._
import docspell.query.internal.{Constants => C}
object SimpleExprParser {
@@ -11,7 +12,7 @@ object SimpleExprParser {
OperatorParser.op.surroundedBy(BasicParser.ws0)
private[this] val inOp: P[Unit] =
P.string("~=").surroundedBy(BasicParser.ws0)
P.string(C.in).surroundedBy(BasicParser.ws0)
private[this] val inOrOpStr =
P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore)
@@ -44,52 +45,63 @@ object SimpleExprParser {
}
val existsExpr: P[Expr.Exists] =
(P.ignoreCase("exists:") *> AttrParser.anyAttr).map(attr => Expr.Exists(attr))
(P.ignoreCase(C.exist) *> P.char(C.like) *> AttrParser.anyAttr).map(attr =>
Expr.Exists(attr)
)
val fulltextExpr: P[Expr.Fulltext] =
(P.ignoreCase("content:") *> BasicParser.singleString).map(q => Expr.Fulltext(q))
(P.ignoreCase(C.content) *> P.char(C.like) *> BasicParser.singleString).map(q =>
Expr.Fulltext(q)
)
val tagIdExpr: P[Expr.TagIdsMatch] =
(P.ignoreCase("tag.id") *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
(P.ignoreCase(C.tagId) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
case (op, values) =>
Expr.TagIdsMatch(op, values)
}
val tagExpr: P[Expr.TagsMatch] =
(P.ignoreCase("tag") *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
(P.ignoreCase(C.tag) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
case (op, values) =>
Expr.TagsMatch(op, values)
}
val catExpr: P[Expr.TagCategoryMatch] =
(P.ignoreCase("cat") *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
(P.ignoreCase(C.cat) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map {
case (op, values) =>
Expr.TagCategoryMatch(op, values)
}
val customFieldExpr: P[Expr.CustomFieldMatch] =
(P.string("f:") *> BasicParser.identParser ~ op ~ BasicParser.singleString).map {
case ((name, op), value) =>
(P.string(C.customField) *> P.char(
C.like
) *> BasicParser.identParser ~ op ~ BasicParser.singleString)
.map { case ((name, op), value) =>
Expr.CustomFieldMatch(name, op, value)
}
}
val customFieldIdExpr: P[Expr.CustomFieldIdMatch] =
(P.string("f.id:") *> BasicParser.identParser ~ op ~ BasicParser.singleString).map {
case ((name, op), value) =>
(P.string(C.customFieldId) *> P.char(
C.like
) *> BasicParser.identParser ~ op ~ BasicParser.singleString)
.map { case ((name, op), value) =>
Expr.CustomFieldIdMatch(name, op, value)
}
}
val inboxExpr: P[Expr.InboxExpr] =
(P.string("inbox:") *> BasicParser.bool).map(Expr.InboxExpr.apply)
(P.string(C.inbox) *> P.char(C.like) *> BasicParser.bool).map(Expr.InboxExpr.apply)
val dirExpr: P[Expr.DirectionExpr] =
(P.string("incoming:") *> BasicParser.bool).map(Expr.DirectionExpr.apply)
(P.string(C.incoming) *> P.char(C.like) *> BasicParser.bool)
.map(Expr.DirectionExpr.apply)
val checksumExpr: P[Expr.ChecksumMatch] =
(P.string("checksum:") *> BasicParser.singleString).map(Expr.ChecksumMatch.apply)
(P.string(C.checksum) *> P.char(C.like) *> BasicParser.singleString)
.map(Expr.ChecksumMatch.apply)
val attachIdExpr: P[Expr.AttachId] =
(P.ignoreCase("attach.id:") *> BasicParser.singleString).map(Expr.AttachId.apply)
(P.ignoreCase(C.attachId) *> P.char(C.eqs) *> BasicParser.singleString)
.map(Expr.AttachId.apply)
val simpleExpr: P[Expr] =
P.oneOf(

4
modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala
View File

@@ -42,9 +42,9 @@ class FulltextExtractTest extends FunSuite {
test("find fulltext within and") {
assertFtsSuccess("content:what name:test", "what".some)
assertFtsSuccess("$names:marc* content:what name:test", "what".some)
assertFtsSuccess("names:marc* content:what name:test", "what".some)
assertFtsSuccess(
"$names:marc* date:2021-02 content:\"what else\" name:test",
"names:marc* date:2021-02 content:\"what else\" name:test",
"what else".some
)
}

4
modules/query/shared/src/test/scala/docspell/query/internal/AttrParserTest.scala
View File

@@ -13,10 +13,8 @@ class AttrParserTest extends FunSuite {
assertEquals(p.parseAll("id"), Right(Attr.ItemId))
assertEquals(p.parseAll("corr.org.id"), Right(Attr.Correspondent.OrgId))
assertEquals(p.parseAll("corr.org.name"), Right(Attr.Correspondent.OrgName))
assertEquals(p.parseAll("correspondent.org.name"), Right(Attr.Correspondent.OrgName))
assertEquals(p.parseAll("conc.pers.id"), Right(Attr.Concerning.PersonId))
assertEquals(p.parseAll("conc.pers.name"), Right(Attr.Concerning.PersonName))
assertEquals(p.parseAll("concerning.person.name"), Right(Attr.Concerning.PersonName))
assertEquals(p.parseAll("folder"), Right(Attr.Folder.FolderName))
assertEquals(p.parseAll("folder.id"), Right(Attr.Folder.FolderId))
}
@@ -24,14 +22,12 @@ class AttrParserTest extends FunSuite {
test("date attributes") {
val p = AttrParser.dateAttr
assertEquals(p.parseAll("date"), Right(Attr.Date))
assertEquals(p.parseAll("dueDate"), Right(Attr.DueDate))
assertEquals(p.parseAll("due"), Right(Attr.DueDate))
}
test("all attributes parser") {
val p = AttrParser.anyAttr
assertEquals(p.parseAll("date"), Right(Attr.Date))
assertEquals(p.parseAll("dueDate"), Right(Attr.DueDate))
assertEquals(p.parseAll("name"), Right(Attr.ItemName))
assertEquals(p.parseAll("source"), Right(Attr.ItemSource))
assertEquals(p.parseAll("id"), Right(Attr.ItemId))

6
modules/query/shared/src/test/scala/docspell/query/internal/MacroParserTest.scala
View File

@@ -6,10 +6,10 @@ import docspell.query.ItemQuery.Expr
class MacroParserTest extends FunSuite {
test("start with $") {
test("recognize names shortcut") {
val p = MacroParser.namesMacro
assertEquals(p.parseAll("$names:test"), Right(Expr.NamesMacro("test")))
assert(p.parseAll("names:test").isLeft)
assertEquals(p.parseAll("names:test"), Right(Expr.NamesMacro("test")))
assert(p.parseAll("$names:test").isLeft)
}
}

8
modules/query/shared/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala
View File

@@ -77,10 +77,10 @@ class SimpleExprParserTest extends FunSuite with ValueHelper {
test("exists expr") {
val p = SimpleExprParser.existsExpr
assertEquals(p.parseAll("exists:name"), Right(Expr.Exists(Attr.ItemName)))
assert(p.parseAll("exists:blabla").isLeft)
assertEquals(p.parseAll("exist:name"), Right(Expr.Exists(Attr.ItemName)))
assert(p.parseAll("exist:blabla").isLeft)
assertEquals(
p.parseAll("exists:conc.pers.id"),
p.parseAll("exist:conc.pers.id"),
Right(Expr.Exists(Attr.Concerning.PersonId))
)
}
@@ -153,7 +153,7 @@ class SimpleExprParserTest extends FunSuite with ValueHelper {
Right(dateExpr(Operator.Lt, Attr.DueDate, ld(2021, 3, 14)))
)
assertEquals(
p.parseAll("exists:conc.pers.id"),
p.parseAll("exist:conc.pers.id"),
Right(Expr.Exists(Attr.Concerning.PersonId))
)
assertEquals(p.parseAll("content:test"), Right(Expr.Fulltext("test")))

3
modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala
View File

@@ -35,7 +35,8 @@ object ItemQueryGeneratorTest extends SimpleTestSuite {
val cond = ItemQueryGenerator(now, tables, Ident.unsafe("coll"))(q)
val expect =
tables.item.name.like("hello") &&
tables.item.itemDate >= mkTimestamp(2020, 2, 1) &&
coalesce(tables.item.itemDate.s, tables.item.created.s) >=
mkTimestamp(2020, 2, 1) &&
(tables.item.source.like("expense%") || tables.folder.name === "test")
assertEquals(cond, expect)

8
modules/webapp/src/main/elm/Data/ItemQuery.elm
View File

@@ -140,16 +140,16 @@ render q =
"folder.id" ++ attrMatch m ++ quoteStr id
CorrOrgId m id ->
"correspondent.org.id" ++ attrMatch m ++ quoteStr id
"corr.org.id" ++ attrMatch m ++ quoteStr id
CorrPersId m id ->
"correspondent.person.id" ++ attrMatch m ++ quoteStr id
"corr.pers.id" ++ attrMatch m ++ quoteStr id
ConcPersId m id ->
"concerning.person.id" ++ attrMatch m ++ quoteStr id
"conc.pers.id" ++ attrMatch m ++ quoteStr id
ConcEquipId m id ->
"concerning.equip.id" ++ attrMatch m ++ quoteStr id
"conc.equip.id" ++ attrMatch m ++ quoteStr id
CustomField m kv ->
"f:" ++ kv.field ++ attrMatch m ++ quoteStr kv.value