Parser improvements

- default expressions into a and node
- fix parsing string lists that end in whitespace
- fix package names of internal classes

build.sbt

@@ -269,6 +269,7 @@ val query =
   crossProject(JSPlatform, JVMPlatform)
     .crossType(CrossType.Pure)
     .in(file("modules/query"))
+    .disablePlugins(RevolverPlugin)
     .settings(sharedSettings)
     .settings(testSettings)
     .settings(
@@ -596,6 +597,7 @@ val website = project
 
 val root = project
   .in(file("."))
+  .disablePlugins(RevolverPlugin)
   .settings(sharedSettings)
   .settings(noPublish)
   .settings(

modules/query/src/main/scala/docspell/query/ItemQuery.scala

@@ -14,10 +14,12 @@ import docspell.query.ItemQuery.Attr.{DateAttr, StringAttr}
 final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String])
 
 object ItemQuery {
+  val all = ItemQuery(Expr.Exists(Attr.ItemId), Some(""))
 
   sealed trait Operator
   object Operator {
     case object Eq   extends Operator
+    case object Neq  extends Operator
     case object Like extends Operator
     case object Gt   extends Operator
     case object Lt   extends Operator
@@ -75,24 +77,26 @@ object ItemQuery {
   }
 
   object Expr {
-    case class AndExpr(expr: Nel[Expr]) extends Expr
+    final case class AndExpr(expr: Nel[Expr]) extends Expr
-    case class OrExpr(expr: Nel[Expr])  extends Expr
+    final case class OrExpr(expr: Nel[Expr])  extends Expr
-    case class NotExpr(expr: Expr) extends Expr {
+    final case class NotExpr(expr: Expr) extends Expr {
       override def negate: Expr =
         expr
     }
 
-    case class SimpleExpr(op: Operator, prop: Property)      extends Expr
+    final case class SimpleExpr(op: Operator, prop: Property)      extends Expr
-    case class Exists(field: Attr)                           extends Expr
+    final case class Exists(field: Attr)                           extends Expr
-    case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
+    final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr
+    final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr
 
-    case class TagIdsMatch(op: TagOperator, tags: Nel[String])      extends Expr
+    final case class TagIdsMatch(op: TagOperator, tags: Nel[String])      extends Expr
-    case class TagsMatch(op: TagOperator, tags: Nel[String])        extends Expr
+    final case class TagsMatch(op: TagOperator, tags: Nel[String])        extends Expr
-    case class TagCategoryMatch(op: TagOperator, cats: Nel[String]) extends Expr
+    final case class TagCategoryMatch(op: TagOperator, cats: Nel[String]) extends Expr
 
-    case class CustomFieldMatch(name: String, op: Operator, value: String) extends Expr
+    final case class CustomFieldMatch(name: String, op: Operator, value: String)
+        extends Expr
 
-    case class Fulltext(query: String) extends Expr
+    final case class Fulltext(query: String) extends Expr
   }
 
 }

modules/query/src/main/scala/docspell/query/ItemQueryParser.scala

@@ -3,17 +3,22 @@ package docspell.query
 import scala.scalajs.js.annotation._
 
 import docspell.query.internal.ExprParser
+import docspell.query.internal.ExprUtil
 
 @JSExportTopLevel("DsItemQueryParser")
 object ItemQueryParser {
 
   @JSExport
   def parse(input: String): Either[String, ItemQuery] =
-    ExprParser.exprParser
+    if (input.isEmpty) Right(ItemQuery.all)
-      .parseAll(input.trim)
+    else {
+      val in = if (input.charAt(0) == '(') input else s"(& $input )"
+      ExprParser
+        .parseQuery(in)
         .left
-      .map(pe => s"Error parsing: '${input.trim}': $pe")
+        .map(pe => s"Error parsing: '$input': $pe")
-      .map(expr => ItemQuery(expr, Some(input.trim)))
+        .map(q => q.copy(expr = ExprUtil.reduce(q.expr)))
+    }
 
   def parseUnsafe(input: String): ItemQuery =
     parse(input).fold(sys.error, identity)

modules/query/src/main/scala/docspell/query/internal/BasicParser.scala

@@ -7,17 +7,14 @@ object BasicParser {
   private[this] val whitespace: P[Unit] = P.charIn(" \t\r\n").void
 
   val ws0: Parser0[Unit] = whitespace.rep0.void
-  val ws1: P[Unit]       = whitespace.rep(1).void
+  val ws1: P[Unit]       = whitespace.rep.void
 
-  private[this] val listSep: P[Unit] =
+  val stringListSep: P[Unit] =
-    P.char(',').surroundedBy(BasicParser.ws0).void
+    (ws0.with1.soft ~ P.char(',') ~ ws0).void
-
-  def rep[A](pa: P[A]): P[Nel[A]] =
-    pa.repSep(listSep)
 
   private[this] val basicString: P[String] =
     P.charsWhile(c =>
-      c > ' ' && c != '"' && c != '\\' && c != ',' && c != '[' && c != ']'
+      c > ' ' && c != '"' && c != '\\' && c != ',' && c != '[' && c != ']' && c != '(' && c != ')'
     )
 
   private[this] val identChars: Set[Char] =
@@ -38,14 +35,7 @@ object BasicParser {
   val singleString: P[String] =
     basicString.backtrack.orElse(StringUtil.quoted('"'))
 
-  val stringListValue: P[Nel[String]] = rep(singleString).with1
-    .between(P.char('['), P.char(']'))
-    .backtrack
-    .orElse(rep(singleString))
-
   val stringOrMore: P[Nel[String]] =
-    stringListValue.backtrack.orElse(
+    singleString.repSep(stringListSep)
-      singleString.map(v => Nel.of(v))
-    )
 
 }

modules/query/src/main/scala/docspell/query/internal/DateParser.scala

@@ -1,5 +1,6 @@
 package docspell.query.internal
 
+import cats.data.NonEmptyList
 import cats.implicits._
 import cats.parse.{Numbers, Parser => P}
 
@@ -39,4 +40,7 @@ object DateParser {
   val localDate: P[Date] =
     localDateFromString.backtrack.orElse(dateFromMillis)
 
+  val localDateOrMore: P[NonEmptyList[Date]] =
+    localDate.repSep(BasicParser.stringListSep)
+
 }

modules/query/src/main/scala/docspell/query/internal/ExprParser.scala

@@ -2,6 +2,7 @@ package docspell.query.internal
 
 import cats.parse.{Parser => P}
 
+import docspell.query.ItemQuery
 import docspell.query.ItemQuery._
 
 object ExprParser {
@@ -18,8 +19,8 @@ object ExprParser {
       .between(BasicParser.parenOr, BasicParser.parenClose)
       .map(Expr.OrExpr.apply)
 
-  def not(inner: P[Expr]): P[Expr.NotExpr] =
+  def not(inner: P[Expr]): P[Expr] =
-    (P.char('!') *> inner).map(Expr.NotExpr.apply)
+    (P.char('!') *> inner).map(_.negate)
 
   val exprParser: P[Expr] =
     P.recursive[Expr] { recurse =>
@@ -28,4 +29,9 @@ object ExprParser {
       val notP = not(recurse)
       P.oneOf(SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil)
     }
+
+  def parseQuery(input: String): Either[P.Error, ItemQuery] = {
+    val p = BasicParser.ws0 *> exprParser <* (BasicParser.ws0 ~ P.end)
+    p.parseAll(input).map(expr => ItemQuery(expr, Some(input)))
+  }
 }

modules/query/src/main/scala/docspell/query/internal/ExprUtil.scala

@@ -0,0 +1,50 @@
+package docspell.query.internal
+
+import docspell.query.ItemQuery.Expr._
+import docspell.query.ItemQuery._
+
+object ExprUtil {
+
+  /** Does some basic transformation, like unfolding deeply nested and
+    * trees containing one value etc.
+    */
+  def reduce(expr: Expr): Expr =
+    expr match {
+      case AndExpr(inner) =>
+        if (inner.tail.isEmpty) reduce(inner.head)
+        else AndExpr(inner.map(reduce))
+
+      case OrExpr(inner) =>
+        if (inner.tail.isEmpty) reduce(inner.head)
+        else OrExpr(inner.map(reduce))
+
+      case NotExpr(inner) =>
+        inner match {
+          case NotExpr(inner2) =>
+            reduce(inner2)
+          case _ =>
+            expr
+        }
+
+      case InExpr(_, _) =>
+        expr
+
+      case InDateExpr(_, _) =>
+        expr
+
+      case TagsMatch(_, _) =>
+        expr
+      case TagIdsMatch(_, _) =>
+        expr
+      case Exists(_) =>
+        expr
+      case Fulltext(_) =>
+        expr
+      case SimpleExpr(_, _) =>
+        expr
+      case TagCategoryMatch(_, _) =>
+        expr
+      case CustomFieldMatch(_, _, _) =>
+        expr
+    }
+}

modules/query/src/main/scala/docspell/query/internal/OperatorParser.scala

@@ -8,6 +8,9 @@ object OperatorParser {
   private[this] val Eq: P[Operator] =
     P.char('=').void.map(_ => Operator.Eq)
 
+  private[this] val Neq: P[Operator] =
+    P.string("!=").void.map(_ => Operator.Neq)
+
   private[this] val Like: P[Operator] =
     P.char(':').void.map(_ => Operator.Like)
 
@@ -24,7 +27,7 @@ object OperatorParser {
     P.string("<=").map(_ => Operator.Lte)
 
   val op: P[Operator] =
-    P.oneOf(List(Like, Eq, Gte, Lte, Gt, Lt))
+    P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt))
 
   private[this] val anyOp: P[TagOperator] =
     P.char(':').map(_ => TagOperator.AnyMatch)

modules/query/src/main/scala/docspell/query/internal/SimpleExprParser.scala

@@ -10,15 +10,29 @@ object SimpleExprParser {
   private[this] val op: P[Operator] =
     OperatorParser.op.surroundedBy(BasicParser.ws0)
 
-  val stringExpr: P[Expr.SimpleExpr] =
+  private[this] val inOp: P[Unit] =
-    (AttrParser.stringAttr ~ op ~ BasicParser.singleString).map {
+    P.string("~=").surroundedBy(BasicParser.ws0)
-      case ((attr, op), value) =>
+
+  private[this] val inOrOpStr =
+    P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore)
+
+  private[this] val inOrOpDate =
+    P.eitherOr(op ~ DateParser.localDate, inOp *> DateParser.localDateOrMore)
+
+  val stringExpr: P[Expr] =
+    (AttrParser.stringAttr ~ inOrOpStr).map {
+      case (attr, Right((op, value))) =>
         Expr.SimpleExpr(op, Property.StringProperty(attr, value))
+      case (attr, Left(values)) =>
+        Expr.InExpr(attr, values)
     }
 
-  val dateExpr: P[Expr.SimpleExpr] =
+  val dateExpr: P[Expr] =
-    (AttrParser.dateAttr ~ op ~ DateParser.localDate).map { case ((attr, op), value) =>
+    (AttrParser.dateAttr ~ inOrOpDate).map {
+      case (attr, Right((op, value))) =>
         Expr.SimpleExpr(op, Property.DateProperty(attr, value))
+      case (attr, Left(values)) =>
+        Expr.InDateExpr(attr, values)
     }
 
   val existsExpr: P[Expr.Exists] =

modules/query/src/test/scala/docspell/query/internal/AttrParserTest.scala

@@ -1,4 +1,4 @@
-package docspell.query
+package docspell.query.internal
 
 import docspell.query.ItemQuery.Attr
 import docspell.query.internal.AttrParser

modules/query/src/test/scala/docspell/query/internal/BasicParserTest.scala

@@ -1,4 +1,4 @@
-package docspell.query
+package docspell.query.internal
 
 import minitest._
 import cats.data.{NonEmptyList => Nel}
@@ -14,13 +14,7 @@ object BasicParserTest extends SimpleTestSuite {
   }
 
   test("string list values") {
-    val p = BasicParser.stringListValue
+    val p = BasicParser.stringOrMore
-    assertEquals(p.parseAll("[ab,cd]"), Right(Nel.of("ab", "cd")))
-    assertEquals(p.parseAll("[\"ab 12\",cd]"), Right(Nel.of("ab 12", "cd")))
-    assertEquals(
-      p.parseAll("[\"ab, 12\",cd]"),
-      Right(Nel.of("ab, 12", "cd"))
-    )
     assertEquals(p.parseAll("ab,cd,123"), Right(Nel.of("ab", "cd", "123")))
     assertEquals(p.parseAll("a,b"), Right(Nel.of("a", "b")))
     assert(p.parseAll("[a,b").isLeft)
@@ -30,6 +24,7 @@ object BasicParserTest extends SimpleTestSuite {
     val p = BasicParser.stringOrMore
     assertEquals(p.parseAll("abcde"), Right(Nel.of("abcde")))
     assertEquals(p.parseAll(""""a,b,c""""), Right(Nel.of("a,b,c")))
-    assertEquals(p.parseAll("[a,b,c]"), Right(Nel.of("a", "b", "c")))
+
+    assertEquals(p.parse("a, b, c "), Right((" ", Nel.of("a", "b", "c"))))
   }
 }

modules/query/src/test/scala/docspell/query/internal/DateParserTest.scala

@@ -1,7 +1,7 @@
-package docspell.query
+package docspell.query.internal
 
-import docspell.query.internal.DateParser
 import minitest._
+import docspell.query.Date
 
 object DateParserTest extends SimpleTestSuite {
 

modules/query/src/test/scala/docspell/query/internal/ExprParserTest.scala

@@ -1,8 +1,7 @@
-package docspell.query
+package docspell.query.internal
 
 import docspell.query.ItemQuery._
-import docspell.query.SimpleExprParserTest.stringExpr
+import docspell.query.internal.SimpleExprParserTest.stringExpr
-import docspell.query.internal.ExprParser
 import minitest._
 import cats.data.{NonEmptyList => Nel}
 
@@ -45,4 +44,28 @@ object ExprParserTest extends SimpleTestSuite {
       )
     )
   }
+
+  test("tag list inside and/or") {
+    val p = ExprParser.exprParser
+    assertEquals(
+      p.parseAll("(& tag:a,b,c)"),
+      Right(
+        Expr.AndExpr(
+          Nel.of(
+            Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c"))
+          )
+        )
+      )
+    )
+    assertEquals(
+      p.parseAll("(& tag:a,b,c )"),
+      Right(
+        Expr.AndExpr(
+          Nel.of(
+            Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c"))
+          )
+        )
+      )
+    )
+  }
 }

modules/query/src/test/scala/docspell/query/internal/ItemQueryParserTest.scala

@@ -0,0 +1,43 @@
+package docspell.query.internal
+
+import minitest._
+import docspell.query.ItemQueryParser
+import docspell.query.ItemQuery
+
+object ItemQueryParserTest extends SimpleTestSuite {
+
+  test("reduce ands") {
+    val q    = ItemQueryParser.parseUnsafe("(&(&(&(& name:hello))))")
+    val expr = ExprUtil.reduce(q.expr)
+    assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
+  }
+
+  test("reduce ors") {
+    val q    = ItemQueryParser.parseUnsafe("(|(|(|(| name:hello))))")
+    val expr = ExprUtil.reduce(q.expr)
+    assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
+  }
+
+  test("reduce and/or") {
+    val q    = ItemQueryParser.parseUnsafe("(|(&(&(| name:hello))))")
+    val expr = ExprUtil.reduce(q.expr)
+    assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr)
+  }
+
+  test("reduce inner and/or") {
+    val q    = ItemQueryParser.parseUnsafe("(& name:hello (| name:world))")
+    val expr = ExprUtil.reduce(q.expr)
+    assertEquals(expr, ItemQueryParser.parseUnsafe("(& name:hello name:world)").expr)
+  }
+
+  test("omit and-parens around root structure") {
+    val q      = ItemQueryParser.parseUnsafe("name:hello date>2020-02-02")
+    val expect = ItemQueryParser.parseUnsafe("(& name:hello date>2020-02-02 )")
+    assertEquals(expect, q)
+  }
+
+  test("return all if query is empty") {
+    val q = ItemQueryParser.parseUnsafe("")
+    assertEquals(ItemQuery.all, q)
+  }
+}

modules/query/src/test/scala/docspell/query/internal/OperatorParserTest.scala

@@ -1,4 +1,4 @@
-package docspell.query
+package docspell.query.internal
 
 import minitest._
 import docspell.query.ItemQuery.{Operator, TagOperator}
@@ -8,6 +8,7 @@ object OperatorParserTest extends SimpleTestSuite {
   test("operator values") {
     val p = OperatorParser.op
     assertEquals(p.parseAll("="), Right(Operator.Eq))
+    assertEquals(p.parseAll("!="), Right(Operator.Neq))
     assertEquals(p.parseAll(":"), Right(Operator.Like))
     assertEquals(p.parseAll("<"), Right(Operator.Lt))
     assertEquals(p.parseAll(">"), Right(Operator.Gt))

modules/query/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala

@@ -1,9 +1,9 @@
-package docspell.query
+package docspell.query.internal
 
 import cats.data.{NonEmptyList => Nel}
 import docspell.query.ItemQuery._
-import docspell.query.internal.SimpleExprParser
 import minitest._
+import docspell.query.Date
 
 object SimpleExprParserTest extends SimpleTestSuite {
 
@@ -29,6 +29,11 @@ object SimpleExprParserTest extends SimpleTestSuite {
       p.parseAll("conc.pers.id=Aaiet-aied"),
       Right(stringExpr(Operator.Eq, Attr.Concerning.PersonId, "Aaiet-aied"))
     )
+    assert(p.parseAll("conc.pers.id=Aaiet,aied").isLeft)
+    assertEquals(
+      p.parseAll("name~=hello,world"),
+      Right(Expr.InExpr(Attr.ItemName, Nel.of("hello", "world")))
+    )
   }
 
   test("date expr") {
@@ -41,6 +46,10 @@ object SimpleExprParserTest extends SimpleTestSuite {
       p.parseAll("due<2021-03-14"),
       Right(dateExpr(Operator.Lt, Attr.DueDate, ld(2021, 3, 14)))
     )
+    assertEquals(
+      p.parseAll("due~=2021-03-14,2021-03-13"),
+      Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13))))
+    )
   }
 
   test("exists expr") {

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -51,13 +51,11 @@ object ItemRoutes {
             offset
           ) =>
         val query =
-          q.map(ItemQueryParser.parse) match {
+          ItemQueryParser.parse(q.getOrElse("")) match {
-            case Some(Right(q)) =>
+            case Right(q) =>
               Right(Query(Query.Fix(user.account, None, None), Query.QueryExpr(q)))
-            case Some(Left(err)) =>
+            case Left(err) =>
               Left(err)
-            case None =>
-              Right(Query(Query.Fix(user.account, None, None), Query.QueryForm.empty))
           }
         val li = limit.getOrElse(cfg.maxItemPageSize)
         val of = offset.getOrElse(0)

modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala

@@ -12,6 +12,7 @@ import docspell.store.qb.{Operator => QOp, _}
 import docspell.store.records.{RCustomField, RCustomFieldValue, TagItemName}
 
 import doobie.util.Put
+import docspell.store.queries.QueryWildcard
 
 object ItemQueryGenerator {
 
@@ -80,18 +81,13 @@ object ItemQueryGenerator {
         val col = stringColumn(tables)(attr)
         op match {
           case Operator.Like =>
-            Condition.CompareVal(col, makeOp(op), value.toLowerCase)
+            Condition.CompareVal(col, makeOp(op), QueryWildcard.lower(value))
           case _ =>
             Condition.CompareVal(col, makeOp(op), value)
         }
 
       case Expr.SimpleExpr(op, Property.DateProperty(attr, value)) =>
-        val dt = value match {
+        val dt  = dateToTimestamp(value)
-          case Date.Local(year, month, day) =>
-            Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay())
-          case Date.Millis(ms) =>
-            Timestamp(Instant.ofEpochMilli(ms))
-        }
         val col = timestampColumn(tables)(attr)
         Condition.CompareVal(col, makeOp(op), dt)
 
@@ -100,6 +96,12 @@ object ItemQueryGenerator {
         if (values.tail.isEmpty) col === values.head
         else col.in(values)
 
+      case Expr.InDateExpr(attr, values) =>
+        val col = timestampColumn(tables)(attr)
+        val dts = values.map(dateToTimestamp)
+        if (values.tail.isEmpty) col === dts.head
+        else col.in(dts)
+
       case Expr.TagIdsMatch(op, tags) =>
         val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption)
         NonEmptyList
@@ -140,6 +142,14 @@ object ItemQueryGenerator {
         Condition.unit
     }
 
+  private def dateToTimestamp(date: Date): Timestamp =
+    date match {
+      case Date.Local(year, month, day) =>
+        Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay())
+      case Date.Millis(ms) =>
+        Timestamp(Instant.ofEpochMilli(ms))
+    }
+
   private def anyColumn(tables: Tables)(attr: Attr): Column[_] =
     attr match {
       case s: Attr.StringAttr =>
@@ -177,6 +187,8 @@ object ItemQueryGenerator {
     operator match {
       case Operator.Eq =>
         QOp.Eq
+      case Operator.Neq =>
+        QOp.Neq
       case Operator.Like =>
         QOp.LowerLike
       case Operator.Gt =>