Add support for more generic notification

This is a start to have different kinds of notifications. It is
possible to be notified via e-mail, matrix or gotify. It also extends
the current "periodic query" for due items by allowing notification
over different channels. A "generic periodic query" variant is added
as well.
This commit is contained in:
eikek
2021-11-22 00:22:51 +01:00
parent 93a828720c
commit 4ffc8d1f14
175 changed files with 13041 additions and 599 deletions

View File

@ -0,0 +1,59 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import cats.implicits._
/** The inverse to Parser */
private[jsonminiq] object Format {
def apply(q: JsonMiniQuery): Either[String, String] =
q match {
case JsonMiniQuery.Empty => Right("")
case JsonMiniQuery.Identity => Right("")
case JsonMiniQuery.Fields(fields) =>
Right(fields.toVector.mkString(","))
case JsonMiniQuery.Indexes(nums) =>
Right(nums.toVector.mkString("(", ",", ")"))
case JsonMiniQuery.Filter(values, mt) =>
formatValue(values.head).map(v => formatMatchType(mt) + v)
case JsonMiniQuery.Chain(self, next) =>
for {
s1 <- apply(self)
s2 <- apply(next)
res = next match {
case _: JsonMiniQuery.Fields =>
s1 + "." + s2
case _ =>
s1 + s2
}
} yield res
case JsonMiniQuery.Concat(inner) =>
inner.toVector.traverse(apply).map(_.mkString("[", " | ", "]"))
case JsonMiniQuery.Forall(inner) =>
inner.toVector.traverse(apply).map(_.mkString("[", " & ", "]"))
}
def formatValue(v: String): Either[String, String] =
if (v.forall(Parser.isValidSimpleValue)) Right(v)
else if (v.contains("\"") && v.contains("'"))
Left(s"Value cannot use both \" and ': $v")
else if (v.contains("'")) Right(s"\"$v\"")
else Right(s"'$v'")
def formatMatchType(matchType: JsonMiniQuery.MatchType): String =
matchType match {
case JsonMiniQuery.MatchType.All => "="
case JsonMiniQuery.MatchType.Any => ":"
case JsonMiniQuery.MatchType.None => "!"
}
}

View File

@ -0,0 +1,245 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import cats.Monoid
import cats.data.NonEmptyVector
import cats.implicits._
import io.circe.Decoder
import io.circe.Encoder
import io.circe.Json.Folder
import io.circe.{Json, JsonNumber, JsonObject}
/** Cteate a predicate for a Json value. */
sealed trait JsonMiniQuery { self =>
def apply(json: Json): Vector[Json]
def >>(next: JsonMiniQuery): JsonMiniQuery =
JsonMiniQuery.Chain(self, next)
def ++(other: JsonMiniQuery): JsonMiniQuery =
JsonMiniQuery.Concat(NonEmptyVector.of(self, other))
def thenAny(other: JsonMiniQuery, more: JsonMiniQuery*): JsonMiniQuery =
self >> JsonMiniQuery.or(other, more: _*)
def thenAll(other: JsonMiniQuery, more: JsonMiniQuery*): JsonMiniQuery =
self >> JsonMiniQuery.and(other, more: _*)
def at(field: String, fields: String*): JsonMiniQuery =
self >> JsonMiniQuery.Fields(NonEmptyVector(field, fields.toVector))
def at(index: Int, indexes: Int*): JsonMiniQuery =
self >> JsonMiniQuery.Indexes(NonEmptyVector(index, indexes.toVector))
def isAll(value: String, values: String*): JsonMiniQuery =
self >> JsonMiniQuery.Filter(
NonEmptyVector(value, values.toVector),
JsonMiniQuery.MatchType.All
)
def isAny(value: String, values: String*): JsonMiniQuery =
self >> JsonMiniQuery.Filter(
NonEmptyVector(value, values.toVector),
JsonMiniQuery.MatchType.Any
)
def is(value: String): JsonMiniQuery =
isAny(value)
def &&(other: JsonMiniQuery): JsonMiniQuery =
JsonMiniQuery.and(self, other)
def ||(other: JsonMiniQuery): JsonMiniQuery =
self ++ other
def matches(json: Json): Boolean =
apply(json).nonEmpty
def notMatches(json: Json): Boolean =
!matches(json)
/** Returns a string representation of this that can be parsed back to this value.
* Formatting can fail, because not everything is supported. The idea is that every
* value that was parsed, can be formatted.
*/
def asString: Either[String, String] =
Format(this)
def unsafeAsString: String =
asString.fold(sys.error, identity)
}
object JsonMiniQuery {
def parse(str: String): Either[String, JsonMiniQuery] =
Parser.query
.parseAll(str)
.leftMap(err =>
s"Unexpected input at ${err.failedAtOffset}. Expected: ${err.expected.toList.mkString(", ")}"
)
def unsafeParse(str: String): JsonMiniQuery =
parse(str).fold(sys.error, identity)
val root: JsonMiniQuery = Identity
val id: JsonMiniQuery = Identity
val none: JsonMiniQuery = Empty
def and(self: JsonMiniQuery, more: JsonMiniQuery*): JsonMiniQuery =
Forall(NonEmptyVector(self, more.toVector))
def or(self: JsonMiniQuery, more: JsonMiniQuery*): JsonMiniQuery =
Concat(NonEmptyVector(self, more.toVector))
// --- impl
case object Identity extends JsonMiniQuery {
def apply(json: Json) = Vector(json)
override def >>(next: JsonMiniQuery): JsonMiniQuery = next
}
case object Empty extends JsonMiniQuery {
def apply(json: Json) = Vector.empty
override def at(field: String, fields: String*): JsonMiniQuery = this
override def at(field: Int, fields: Int*): JsonMiniQuery = this
override def isAll(value: String, values: String*) = this
override def isAny(value: String, values: String*) = this
override def >>(next: JsonMiniQuery): JsonMiniQuery = this
override def ++(other: JsonMiniQuery): JsonMiniQuery = other
}
private def unwrapArrays(json: Vector[Json]): Vector[Json] =
json.foldLeft(Vector.empty[Json]) { (res, el) =>
el.asArray.map(x => res ++ x).getOrElse(res :+ el)
}
final case class Fields(names: NonEmptyVector[String]) extends JsonMiniQuery {
def apply(json: Json) = json.foldWith(folder)
private val folder: Folder[Vector[Json]] = new Folder[Vector[Json]] {
def onNull = Vector.empty
def onBoolean(value: Boolean) = Vector.empty
def onNumber(value: JsonNumber) = Vector.empty
def onString(value: String) = Vector.empty
def onArray(value: Vector[Json]) =
unwrapArrays(value.flatMap(inner => inner.foldWith(this)))
def onObject(value: JsonObject) =
unwrapArrays(names.toVector.flatMap(value.apply))
}
}
final case class Indexes(indexes: NonEmptyVector[Int]) extends JsonMiniQuery {
def apply(json: Json) = json.foldWith(folder)
private val folder: Folder[Vector[Json]] = new Folder[Vector[Json]] {
def onNull = Vector.empty
def onBoolean(value: Boolean) = Vector.empty
def onNumber(value: JsonNumber) = Vector.empty
def onString(value: String) = Vector.empty
def onArray(value: Vector[Json]) =
unwrapArrays(indexes.toVector.flatMap(i => value.get(i.toLong)))
def onObject(value: JsonObject) =
Vector.empty
}
}
sealed trait MatchType {
def monoid: Monoid[Boolean]
}
object MatchType {
case object Any extends MatchType {
val monoid = Monoid.instance(false, _ || _)
}
case object All extends MatchType {
val monoid = Monoid.instance(true, _ && _)
}
case object None extends MatchType { // = not Any
val monoid = Monoid.instance(true, _ && !_)
}
}
final case class Filter(
values: NonEmptyVector[String],
combine: MatchType
) extends JsonMiniQuery {
def apply(json: Json): Vector[Json] =
json.asArray match {
case Some(arr) =>
unwrapArrays(arr.filter(el => el.foldWith(folder(combine))))
case None =>
if (json.foldWith(folder(combine))) unwrapArrays(Vector(json))
else Vector.empty
}
private val anyMatch = folder(MatchType.Any)
private def folder(matchType: MatchType): Folder[Boolean] = new Folder[Boolean] {
def onNull =
onString("*null*")
def onBoolean(value: Boolean) =
values
.map(_.equalsIgnoreCase(value.toString))
.fold(matchType.monoid)
def onNumber(value: JsonNumber) =
values
.map(
_.equalsIgnoreCase(
value.toLong.map(_.toString).getOrElse(value.toDouble.toString)
)
)
.fold(matchType.monoid)
def onString(value: String) =
values
.map(_.equalsIgnoreCase(value))
.fold(matchType.monoid)
def onArray(value: Vector[Json]) =
value
.map(inner => inner.foldWith(anyMatch))
.fold(matchType.monoid.empty)(matchType.monoid.combine)
def onObject(value: JsonObject) = false
}
}
final case class Chain(self: JsonMiniQuery, next: JsonMiniQuery) extends JsonMiniQuery {
def apply(json: Json): Vector[Json] =
next(Json.fromValues(self(json)))
}
final case class Concat(qs: NonEmptyVector[JsonMiniQuery]) extends JsonMiniQuery {
def apply(json: Json): Vector[Json] =
qs.toVector.flatMap(_.apply(json))
}
final case class Forall(qs: NonEmptyVector[JsonMiniQuery]) extends JsonMiniQuery {
def apply(json: Json): Vector[Json] =
combineWhenNonEmpty(qs.toVector.map(_.apply(json)), Vector.empty)
@annotation.tailrec
private def combineWhenNonEmpty(
values: Vector[Vector[Json]],
result: Vector[Json]
): Vector[Json] =
values.headOption match {
case Some(v) if v.nonEmpty => combineWhenNonEmpty(values.tail, result ++ v)
case Some(_) => Vector.empty
case None => result
}
}
implicit val jsonDecoder: Decoder[JsonMiniQuery] =
Decoder.decodeString.emap(parse)
implicit val jsonEncoder: Encoder[JsonMiniQuery] =
Encoder.encodeString.contramap(_.unsafeAsString)
}

View File

@ -0,0 +1,105 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import cats.data.NonEmptyVector
import cats.parse.{Parser => P, Parser0 => P0}
import docspell.jsonminiq.JsonMiniQuery.{Identity => JQ}
private[jsonminiq] object Parser {
// a[,b] -> at(string)
// (1[,2,3]) -> at(int)
// :b -> isAny(b)
// =b -> isAll(b)
// [F & G] -> F && G
// [F | G] -> F || G
private[this] val whitespace: P[Unit] = P.charIn(" \t\r\n").void
private[this] val extraFieldChars = "_-".toSet
private[this] val dontUse = "\"'\\[]()&|".toSet
private[this] val ws0: P0[Unit] = whitespace.rep0.void
private[this] val parenOpen: P[Unit] = P.char('(') *> ws0
private[this] val parenClose: P[Unit] = ws0.with1 *> P.char(')')
private[this] val bracketOpen: P[Unit] = P.char('[') *> ws0
private[this] val bracketClose: P[Unit] = ws0.with1 *> P.char(']')
private[this] val dot: P[Unit] = P.char('.')
private[this] val comma: P[Unit] = P.char(',')
private[this] val andSym: P[Unit] = ws0.with1 *> P.char('&') <* ws0
private[this] val orSym: P[Unit] = ws0.with1 *> P.char('|') <* ws0
private[this] val squote: P[Unit] = P.char('\'')
private[this] val dquote: P[Unit] = P.char('"')
private[this] val allOp: P[JsonMiniQuery.MatchType] =
P.char('=').as(JsonMiniQuery.MatchType.All)
private[this] val noneOp: P[JsonMiniQuery.MatchType] =
P.char('!').as(JsonMiniQuery.MatchType.None)
def isValidSimpleValue(c: Char): Boolean =
c > ' ' && !dontUse.contains(c)
val value: P[String] = {
val simpleString: P[String] =
P.charsWhile(isValidSimpleValue)
val quotedString: P[String] = {
val single: P[String] =
squote *> P.charsWhile0(_ != '\'') <* squote
val double: P[String] =
dquote *> P.charsWhile0(_ != '"') <* dquote
single | double
}
simpleString | quotedString
}
val field: P[String] =
P.charsWhile(c => c.isLetterOrDigit || extraFieldChars.contains(c))
val posNum: P[Int] = P.charsWhile(_.isDigit).map(_.toInt).filter(_ >= 0)
val fieldSelect1: P[JsonMiniQuery] =
field.repSep(comma).map(nel => JQ.at(nel.head, nel.tail: _*))
val arraySelect1: P[JsonMiniQuery] = {
val nums = posNum.repSep(1, comma)
parenOpen.soft *> nums.map(f => JQ.at(f.head, f.tail: _*)) <* parenClose
}
val match1: P[JsonMiniQuery] =
((allOp | noneOp) ~ value).map { case (op, v) =>
JsonMiniQuery.Filter(NonEmptyVector.of(v), op)
}
val segment = {
val firstSegment = fieldSelect1 | arraySelect1 | match1
val nextSegment = (dot *> fieldSelect1) | arraySelect1 | match1
(firstSegment ~ nextSegment.rep0).map { case (head, tail) =>
tail.foldLeft(head)(_ >> _)
}
}
def combine(inner: P[JsonMiniQuery]): P[JsonMiniQuery] = {
val or = inner.repSep(orSym).map(_.reduceLeft(_ || _))
val and = inner.repSep(andSym).map(_.reduceLeft(_ && _))
and
.between(bracketOpen, bracketClose)
.backtrack
.orElse(or.between(bracketOpen, bracketClose))
}
val query: P[JsonMiniQuery] =
P.recursive[JsonMiniQuery] { recurse =>
val comb = combine(recurse)
P.oneOf(segment :: comb :: Nil).rep.map(_.reduceLeft(_ >> _))
}
}

View File

@ -0,0 +1,76 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import cats.parse.{Parser => P}
import io.circe.Json
trait Fixtures {
val sampleEvent: Json =
parseJson(
"""{
| "eventType": "TagsChanged",
| "account": {
| "collective": "demo",
| "user": "demo",
| "login": "demo"
| },
| "content": {
| "account": "demo",
| "items": [
| {
| "id": "4PvMM4m7Fwj-FsPRGxYt9zZ-uUzi35S2rEX-usyDEVyheR8",
| "name": "MapleSirupLtd_202331.pdf",
| "dateMillis": 1633557740733,
| "date": "2021-10-06",
| "direction": "incoming",
| "state": "confirmed",
| "dueDateMillis": 1639173740733,
| "dueDate": "2021-12-10",
| "source": "webapp",
| "overDue": false,
| "dueIn": "in 3 days",
| "corrOrg": "Acme AG",
| "notes": null
| }
| ],
| "added": [
| {
| "id": "Fy4VC6hQwcL-oynrHaJg47D-Q5RiQyB5PQP-N5cFJ368c4N",
| "name": "Invoice",
| "category": "doctype"
| },
| {
| "id": "7zaeU6pqVym-6Je3Q36XNG2-ZdBTFSVwNjc-pJRXciTMP3B",
| "name": "Grocery",
| "category": "expense"
| }
| ],
| "removed": [
| {
| "id": "GbXgszdjBt4-zrzuLHoUx7N-RMFatC8CyWt-5dsBCvxaEuW",
| "name": "Receipt",
| "category": "doctype"
| }
| ],
| "itemUrl": "http://localhost:7880/app/item"
| }
|}""".stripMargin
)
def parseJson(str: String): Json =
io.circe.parser.parse(str).fold(throw _, identity)
def parseP[A](p: P[A], str: String): A =
p.parseAll(str.trim())
.fold(e => sys.error(s"${e.getClass}: $e"), identity)
def parse(str: String): JsonMiniQuery = parseP(Parser.query, str)
}

View File

@ -0,0 +1,52 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import docspell.jsonminiq.JsonMiniQuery.{Identity => JQ}
import munit._
class FormatTest extends FunSuite with Fixtures {
def format(q: JsonMiniQuery): String =
q.unsafeAsString
test("field selects") {
assertEquals(
format(JQ.at("content").at("added", "removed").at("name")),
"content.added,removed.name"
)
}
test("array select") {
assertEquals(format(JQ.at("content").at(1, 2).at("name")), "content(1,2).name")
}
test("anyMatch / allMatch") {
assertEquals(format(JQ.isAny("in voice")), ":'in voice'")
assertEquals(format(JQ.isAll("invoice")), "=invoice")
assertEquals(format(JQ.at("name").isAll("invoice")), "name=invoice")
assertEquals(format(JQ.at("name").isAny("invoice")), "name:invoice")
}
test("and / or") {
assertEquals(
format((JQ.at("c") >> JQ.isAll("d")) || (JQ.at("e") >> JQ.isAll("f"))),
"[c=d | e=f]"
)
assertEquals(
format(
(JQ.at("a").isAll("1")) || (
(JQ.at("b").isAll("2")) && (JQ.at("c").isAll("3"))
)
),
"[a=1 | [b=2 & c=3]]"
)
}
}

View File

@ -0,0 +1,161 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import docspell.jsonminiq.JsonMiniQuery.{Identity => JQ}
import io.circe.Encoder
import io.circe.Json
import io.circe.syntax._
import munit._
class JsonMiniQueryTest extends FunSuite with Fixtures {
def values[T: Encoder](v1: T, vn: T*): Vector[Json] =
(v1 +: vn.toVector).map(_.asJson)
test("combine values on same level") {
val q = JQ
.at("content")
.at("added", "removed")
.at("name")
assertEquals(q(sampleEvent), values("Invoice", "Grocery", "Receipt"))
}
test("combine values from different levels") {
val q1 = JQ.at("account")
val q2 = JQ.at("removed").at("name")
val q = JQ.at("content") >> (q1 ++ q2)
assertEquals(q(sampleEvent), values("demo", "Receipt"))
}
test("filter single value") {
val q = JQ.at("account").at("login").isAll("demo")
assertEquals(q(sampleEvent), values("demo"))
val q2 = JQ.at("account").at("login").isAll("james")
assertEquals(q2(sampleEvent), Vector.empty)
}
test("combine filters") {
val q1 = JQ.at("account").at("login").isAll("demo")
val q2 = JQ.at("eventType").isAll("tagschanged")
val q3 = JQ.at("content").at("added", "removed").at("name").isAny("invoice")
val q = q1 && q2 && q3
assertEquals(
q(sampleEvent),
values("demo", "TagsChanged", "Invoice")
)
val q11 = JQ.at("account").at("login").isAll("not-exists")
val r = q11 && q2 && q3
assertEquals(r(sampleEvent), Vector.empty)
}
//content.[added,removed].(category=expense & name=grocery)
test("combine fields and filter") {
val andOk = JQ.at("content").at("added", "removed") >>
(JQ.at("name").is("grocery") && JQ.at("category").is("expense"))
assert(andOk.matches(sampleEvent))
val andNotOk = JQ.at("content").at("added", "removed") >>
(JQ.at("name").is("grocery") && JQ.at("category").is("notexist"))
assert(andNotOk.notMatches(sampleEvent))
val orOk = JQ.at("content").at("added", "removed") >>
(JQ.at("name").is("grocery") || JQ.at("category").is("notexist"))
assert(orOk.matches(sampleEvent))
}
test("thenAny combine via or") {
val q = JQ
.at("content")
.thenAny(
JQ.is("not this"),
JQ.at("account"),
JQ.at("oops")
)
assert(q.matches(sampleEvent))
}
test("thenAll combine via and (1)") {
val q = JQ
.at("content")
.thenAll(
JQ.is("not this"),
JQ.at("account"),
JQ.at("oops")
)
assert(q.notMatches(sampleEvent))
}
test("thenAll combine via and (2)") {
val q = JQ
.at("content")
.thenAll(
JQ.at("items").at("date").is("2021-10-06"),
JQ.at("account"),
JQ.at("added").at("name")
)
assert(q.matches(sampleEvent))
// equivalent
val q2 = JQ.at("content") >> (
JQ.at("items").at("date").is("2021-10-06") &&
JQ.at("account") &&
JQ.at("added").at("name")
)
assert(q2.matches(sampleEvent))
}
test("test for null/not null") {
val q1 = parse("content.items.notes=*null*")
assert(q1.matches(sampleEvent))
val q2 = parse("content.items.notes=bla")
assert(q2.notMatches(sampleEvent))
val q3 = parse("content.items.notes!*null*")
assert(q3.notMatches(sampleEvent))
}
test("more real expressions") {
val q = parse("content.added,removed[name=invoice | category=expense]")
assert(q.matches(sampleEvent))
}
test("examples") {
val q0 = parse("a.b.x,y")
val json = parseJson(
"""[{"a": {"b": {"x": 1, "y":2}}, "v": 0}, {"a": {"b": {"y": 9, "b": 2}}, "z": 0}]"""
)
assertEquals(q0(json), values(1, 2, 9))
val q1 = parse("a(0,2)")
val json1 = parseJson("""[{"a": [10,9,8,7]}, {"a": [1,2,3,4]}]""")
assertEquals(q1(json1), values(10, 8))
val q2 = parse("=blue")
val json2 = parseJson("""["blue", "green", "red"]""")
assertEquals(q2(json2), values("blue"))
val q3 = parse("color=blue")
val json3 = parseJson(
"""[{"color": "blue", "count": 2}, {"color": "blue", "count": 1}, {"color": "blue", "count": 3}]"""
)
assertEquals(q3(json3), values("blue", "blue", "blue"))
val q4 = parse("[count=6 | name=max]")
val json4 = parseJson(
"""[{"name":"max", "count":4}, {"name":"me", "count": 3}, {"name":"max", "count": 3}]"""
)
println(q4(json4))
}
}

View File

@ -0,0 +1,54 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.jsonminiq
import docspell.jsonminiq.JsonMiniQuery.{Identity => JQ}
import munit._
class ParserTest extends FunSuite with Fixtures {
test("field selects") {
assertEquals(
parse("content.added,removed.name"),
JQ.at("content").at("added", "removed").at("name")
)
}
test("array select") {
assertEquals(parse("content(1,2).name"), JQ.at("content").at(1, 2).at("name"))
}
test("values") {
assertEquals(parseP(Parser.value, "\"in voice\""), "in voice")
assertEquals(parseP(Parser.value, "'in voice'"), "in voice")
assertEquals(parseP(Parser.value, "invoice"), "invoice")
intercept[Throwable](parseP(Parser.value, "in voice"))
}
test("anyMatch / allMatch") {
assertEquals(parse("='invoice'"), JQ.isAll("invoice"))
assertEquals(parse("=invoice"), JQ.isAll("invoice"))
assertEquals(parse("name=invoice"), JQ.at("name").isAll("invoice"))
assertEquals(parse("name=\"invoice\""), JQ.at("name").isAll("invoice"))
}
test("and / or") {
assertEquals(
parse("[c=d | e=f]"),
(JQ.at("c") >> JQ.isAll("d")) || (JQ.at("e") >> JQ.isAll("f"))
)
assertEquals(
parse("[a=1 | [b=2 & c=3]]"),
(JQ.at("a") >> JQ.isAll("1")) || (
(JQ.at("b") >> JQ.isAll("2")) && (JQ.at("c") >> JQ.isAll("3"))
)
)
}
}