diff --git a/build.sbt b/build.sbt index 2c9db4f1..3a1ca00a 100644 --- a/build.sbt +++ b/build.sbt @@ -47,7 +47,13 @@ val sharedSettings = Seq( val testSettings = Seq( testFrameworks += new TestFramework("minitest.runner.Framework"), - libraryDependencies ++= Dependencies.miniTest ++ Dependencies.logging.map(_ % Test) + libraryDependencies ++= Dependencies.miniTest ++ Dependencies.logging.map(_ % Test), + Test / fork := true +) + +val testSettingsMUnit = Seq( + libraryDependencies ++= Dependencies.munit.map(_ % Test), + testFrameworks += new TestFramework("munit.Framework") ) lazy val noPublish = Seq( @@ -80,7 +86,7 @@ val stylesSettings = Seq( Compile / resourceGenerators += stylesBuild.taskValue ) -val webjarSettings = Seq( +def webjarSettings(queryJS: Project) = Seq( Compile / resourceGenerators += Def.task { copyWebjarResources( Seq((sourceDirectory in Compile).value / "webjar"), @@ -90,6 +96,18 @@ val webjarSettings = Seq( streams.value.log ) }.taskValue, + Compile / resourceGenerators += Def.task { + val logger = streams.value.log + val out = (queryJS/Compile/fullOptJS).value + logger.info(s"Produced query js file: ${out.data}") + copyWebjarResources( + Seq(out.data), + (Compile/resourceManaged).value, + name.value, + version.value, + logger + ) + }.taskValue, watchSources += Watched.WatchSource( (Compile / sourceDirectory).value / "webjar", FileFilter.globFilter("*.js") || FileFilter.globFilter("*.css"), @@ -264,6 +282,28 @@ ${lines.map(_._1).mkString(",\n")} ) .dependsOn(common) +val query = + crossProject(JSPlatform, JVMPlatform) + .withoutSuffixFor(JVMPlatform) + .in(file("modules/query")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettingsMUnit) + .settings( + name := "docspell-query", + libraryDependencies += + Dependencies.catsParseJS.value, + libraryDependencies += + Dependencies.scalaJavaTime.value + ) + .jsSettings( + Test / fork := false + ) + .jvmSettings( + libraryDependencies += + Dependencies.scalaJsStubs + ) + val store = project .in(file("modules/store")) .disablePlugins(RevolverPlugin) @@ -284,7 +324,7 @@ val store = project Dependencies.calevCore ++ Dependencies.calevFs2 ) - .dependsOn(common) + .dependsOn(common, query.jvm) val extract = project .in(file("modules/extract")) @@ -417,7 +457,7 @@ val webapp = project .settings(sharedSettings) .settings(elmSettings) .settings(stylesSettings) - .settings(webjarSettings) + .settings(webjarSettings(query.js)) .settings( name := "docspell-webapp", openapiTargetLanguage := Language.Elm, @@ -425,6 +465,7 @@ val webapp = project openapiSpec := (restapi / Compile / resourceDirectory).value / "docspell-openapi.yml", openapiElmConfig := ElmConfig().withJson(ElmJson.decodePipeline) ) + .dependsOn(query.js) // --- Application(s) @@ -575,6 +616,7 @@ val website = project val root = project .in(file(".")) + .disablePlugins(RevolverPlugin) .settings(sharedSettings) .settings(noPublish) .settings( @@ -594,7 +636,9 @@ val root = project backend, webapp, restapi, - restserver + restserver, + query.jvm, + query.js ) // --- Helpers diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 81328296..80df397c 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -37,6 +37,7 @@ trait BackendApp[F[_]] { def userTask: OUserTask[F] def folder: OFolder[F] def customFields: OCustomFields[F] + def simpleSearch: OSimpleSearch[F] } object BackendApp { @@ -71,6 +72,7 @@ object BackendApp { userTaskImpl <- OUserTask(utStore, queue, joexImpl) folderImpl <- OFolder(store) customFieldsImpl <- OCustomFields(store) + simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl) } yield new BackendApp[F] { val login = loginImpl val signup = signupImpl @@ -90,6 +92,7 @@ object BackendApp { val userTask = userTaskImpl val folder = folderImpl val customFields = customFieldsImpl + val simpleSearch = simpleSearchImpl } def apply[F[_]: ConcurrentEffect: ContextShift]( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 52e23571..0dd2348c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -159,13 +159,14 @@ object OFulltext { for { folder <- store.transact(QFolder.getMemberFolders(account)) + now <- Timestamp.current[F] itemIds <- fts .searchAll(fq.withFolders(folder)) .flatMap(r => Stream.emits(r.results.map(_.itemId))) .compile .to(Set) - q = Query.empty(account).copy(itemIds = itemIds.some) - res <- store.transact(QItem.searchStats(q)) + q = Query.empty(account).withFix(_.copy(itemIds = itemIds.some)) + res <- store.transact(QItem.searchStats(now.toUtcDate)(q)) } yield res } @@ -208,7 +209,7 @@ object OFulltext { search <- itemSearch.findItems(0)(q, Batch.all) fq = FtsQuery( ftsQ.query, - q.account.collective, + q.fix.account.collective, search.map(_.id).toSet, Set.empty, 500, @@ -220,8 +221,9 @@ object OFulltext { .flatMap(r => Stream.emits(r.results.map(_.itemId))) .compile .to(Set) - qnext = q.copy(itemIds = items.some) - res <- store.transact(QItem.searchStats(qnext)) + qnext = q.withFix(_.copy(itemIds = items.some)) + now <- Timestamp.current[F] + res <- store.transact(QItem.searchStats(now.toUtcDate)(qnext)) } yield res // Helper @@ -253,7 +255,7 @@ object OFulltext { val sqlResult = search(q, batch) val fq = FtsQuery( ftsQ.query, - q.account.collective, + q.fix.account.collective, Set.empty, Set.empty, 0, diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 46ec929d..a74e451a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -127,25 +127,39 @@ object OItemSearch { .map(opt => opt.flatMap(_.filterCollective(collective))) def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] = - store - .transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong)) - .compile - .toVector + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap { today => + store + .transact( + QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong) + ) + .compile + .toVector + } def findItemsWithTags( maxNoteLen: Int - )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = { - val search = QItem.findItems(q, maxNoteLen: Int, batch) - store - .transact( - QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong) - ) - .compile - .toVector - } + )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = + for { + now <- Timestamp.current[F] + search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch) + res <- store + .transact( + QItem + .findItemsWithTags(q.fix.account.collective, search) + .take(batch.limit.toLong) + ) + .compile + .toVector + } yield res def findItemsSummary(q: Query): F[SearchSummary] = - store.transact(QItem.searchStats(q)) + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap(today => store.transact(QItem.searchStats(today)(q))) def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala new file mode 100644 index 00000000..1c5e54df --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala @@ -0,0 +1,221 @@ +package docspell.backend.ops + +import cats.effect.Sync +import cats.implicits._ + +import docspell.backend.ops.OSimpleSearch._ +import docspell.common._ +import docspell.query._ +import docspell.store.qb.Batch +import docspell.store.queries.Query +import docspell.store.queries.SearchSummary + +/** A "porcelain" api on top of OFulltext and OItemSearch. */ +trait OSimpleSearch[F[_]] { + + def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items] + def searchSummary( + useFTS: Boolean + )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] + + def searchByString( + settings: Settings + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] + def searchSummaryByString( + useFTS: Boolean + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] + +} + +object OSimpleSearch { + + sealed trait StringSearchResult[+A] + object StringSearchResult { + case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing] + def parseFailed[A](error: ParseFailure): StringSearchResult[A] = + ParseFailed(error) + + case class FulltextMismatch(error: FulltextExtract.FailureResult) + extends StringSearchResult[Nothing] + def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] = + FulltextMismatch(error) + + case class Success[A](value: A) extends StringSearchResult[A] + } + + final case class Settings( + batch: Batch, + useFTS: Boolean, + resolveDetails: Boolean, + maxNoteLen: Int + ) + + sealed trait Items { + def fold[A]( + f1: Items.FtsItems => A, + f2: Items.FtsItemsFull => A, + f3: Vector[OItemSearch.ListItem] => A, + f4: Vector[OItemSearch.ListItemWithTags] => A + ): A + + } + object Items { + def ftsItems(indexOnly: Boolean)(items: Vector[OFulltext.FtsItem]): Items = + FtsItems(items, indexOnly) + + case class FtsItems(items: Vector[OFulltext.FtsItem], indexOnly: Boolean) + extends Items { + def fold[A]( + f1: FtsItems => A, + f2: FtsItemsFull => A, + f3: Vector[OItemSearch.ListItem] => A, + f4: Vector[OItemSearch.ListItemWithTags] => A + ): A = f1(this) + + } + + def ftsItemsFull(indexOnly: Boolean)( + items: Vector[OFulltext.FtsItemWithTags] + ): Items = + FtsItemsFull(items, indexOnly) + + case class FtsItemsFull(items: Vector[OFulltext.FtsItemWithTags], indexOnly: Boolean) + extends Items { + def fold[A]( + f1: FtsItems => A, + f2: FtsItemsFull => A, + f3: Vector[OItemSearch.ListItem] => A, + f4: Vector[OItemSearch.ListItemWithTags] => A + ): A = f2(this) + } + + def itemsPlain(items: Vector[OItemSearch.ListItem]): Items = + ItemsPlain(items) + + case class ItemsPlain(items: Vector[OItemSearch.ListItem]) extends Items { + def fold[A]( + f1: FtsItems => A, + f2: FtsItemsFull => A, + f3: Vector[OItemSearch.ListItem] => A, + f4: Vector[OItemSearch.ListItemWithTags] => A + ): A = f3(items) + } + + def itemsFull(items: Vector[OItemSearch.ListItemWithTags]): Items = + ItemsFull(items) + + case class ItemsFull(items: Vector[OItemSearch.ListItemWithTags]) extends Items { + def fold[A]( + f1: FtsItems => A, + f2: FtsItemsFull => A, + f3: Vector[OItemSearch.ListItem] => A, + f4: Vector[OItemSearch.ListItemWithTags] => A + ): A = f4(items) + } + + } + + def apply[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]): OSimpleSearch[F] = + new Impl(fts, is) + + final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]) + extends OSimpleSearch[F] { + def searchByString( + settings: Settings + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] = { + val parsed: Either[StringSearchResult[Items], ItemQuery] = + ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed) + + def makeQuery(iq: ItemQuery): F[StringSearchResult[Items]] = + iq.findFulltext match { + case FulltextExtract.Result.Success(expr, ftq) => + search(settings)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + .map(StringSearchResult.Success.apply) + case other: FulltextExtract.FailureResult => + StringSearchResult.fulltextMismatch[Items](other).pure[F] + } + + parsed match { + case Right(iq) => + makeQuery(iq) + case Left(err) => + err.pure[F] + } + } + + def searchSummaryByString( + useFTS: Boolean + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] = { + val parsed: Either[StringSearchResult[SearchSummary], ItemQuery] = + ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed) + + def makeQuery(iq: ItemQuery): F[StringSearchResult[SearchSummary]] = + iq.findFulltext match { + case FulltextExtract.Result.Success(expr, ftq) => + searchSummary(useFTS)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + .map(StringSearchResult.Success.apply) + case other: FulltextExtract.FailureResult => + StringSearchResult.fulltextMismatch[SearchSummary](other).pure[F] + } + + parsed match { + case Right(iq) => + makeQuery(iq) + case Left(err) => + err.pure[F] + } + } + + def searchSummary( + useFTS: Boolean + )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = + fulltextQuery match { + case Some(ftq) if useFTS => + if (q.isEmpty) + fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq)) + else + fts + .findItemsSummary(q, OFulltext.FtsInput(ftq)) + + case _ => + is.findItemsSummary(q) + } + + def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items] = + // 1. fulltext only if fulltextQuery.isDefined && q.isEmpty && useFTS + // 2. sql+fulltext if fulltextQuery.isDefined && q.nonEmpty && useFTS + // 3. sql-only else (if fulltextQuery.isEmpty || !useFTS) + fulltextQuery match { + case Some(ftq) if settings.useFTS => + if (q.isEmpty) + fts + .findIndexOnly(settings.maxNoteLen)( + OFulltext.FtsInput(ftq), + q.fix.account, + settings.batch + ) + .map(Items.ftsItemsFull(true)) + else if (settings.resolveDetails) + fts + .findItemsWithTags(settings.maxNoteLen)( + q, + OFulltext.FtsInput(ftq), + settings.batch + ) + .map(Items.ftsItemsFull(false)) + else + fts + .findItems(settings.maxNoteLen)(q, OFulltext.FtsInput(ftq), settings.batch) + .map(Items.ftsItems(false)) + + case _ => + if (settings.resolveDetails) + is.findItemsWithTags(settings.maxNoteLen)(q, settings.batch) + .map(Items.itemsFull) + else + is.findItems(settings.maxNoteLen)(q, settings.batch) + .map(Items.itemsPlain) + } + } + +} diff --git a/modules/common/src/main/scala/docspell/common/ItemQueryString.scala b/modules/common/src/main/scala/docspell/common/ItemQueryString.scala new file mode 100644 index 00000000..8e6e887e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ItemQueryString.scala @@ -0,0 +1,9 @@ +package docspell.common + +case class ItemQueryString(query: String) + +object ItemQueryString { + + def apply(qs: Option[String]): ItemQueryString = + ItemQueryString(qs.getOrElse("")) +} diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index e31b6fd7..4ce26507 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -72,17 +72,24 @@ object NotifyDueItemsTask { q = Query .empty(ctx.args.account) - .copy( - states = ItemState.validStates.toList, - tagsInclude = ctx.args.tagsInclude, - tagsExclude = ctx.args.tagsExclude, - dueDateFrom = ctx.args.daysBack.map(back => now - Duration.days(back.toLong)), - dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong)), - orderAsc = Some(_.dueDate) + .withOrder(orderAsc = _.dueDate) + .withCond(_ => + Query.QueryForm.empty.copy( + states = ItemState.validStates.toList, + tagsInclude = ctx.args.tagsInclude, + tagsExclude = ctx.args.tagsExclude, + dueDateFrom = + ctx.args.daysBack.map(back => now - Duration.days(back.toLong)), + dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong)) + ) ) res <- ctx.store - .transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong)) + .transact( + QItem + .findItems(q, now.toUtcDate, 0, Batch.limit(maxItems)) + .take(maxItems.toLong) + ) .compile .toVector } yield res diff --git a/modules/query/js/src/main/scala/docspell/query/js/JSItemQueryParser.scala b/modules/query/js/src/main/scala/docspell/query/js/JSItemQueryParser.scala new file mode 100644 index 00000000..a58371cd --- /dev/null +++ b/modules/query/js/src/main/scala/docspell/query/js/JSItemQueryParser.scala @@ -0,0 +1,29 @@ +package docspell.query.js + +import scala.scalajs.js +import scala.scalajs.js.annotation._ + +import docspell.query.ItemQueryParser + +@JSExportTopLevel("DsItemQueryParser") +object JSItemQueryParser { + + @JSExport + def parseToFailure(input: String): Failure = + ItemQueryParser + .parse(input) + .swap + .toOption + .map(fr => + new Failure( + fr.input, + fr.failedAt, + js.Array(fr.messages.toList.toSeq.map(_.render): _*) + ) + ) + .orNull + + @JSExportAll + case class Failure(input: String, failedAt: Int, messages: js.Array[String]) + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/Date.scala b/modules/query/shared/src/main/scala/docspell/query/Date.scala new file mode 100644 index 00000000..21ce9b35 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/Date.scala @@ -0,0 +1,32 @@ +package docspell.query + +import java.time.LocalDate +import java.time.Period + +import cats.implicits._ + +sealed trait Date + +object Date { + def apply(y: Int, m: Int, d: Int): Either[Throwable, DateLiteral] = + Either.catchNonFatal(Local(LocalDate.of(y, m, d))) + + def apply(ms: Long): DateLiteral = + Millis(ms) + + sealed trait DateLiteral extends Date + + final case class Local(date: LocalDate) extends DateLiteral + + final case class Millis(ms: Long) extends DateLiteral + + case object Today extends DateLiteral + + sealed trait CalcDirection + object CalcDirection { + case object Plus extends CalcDirection + case object Minus extends CalcDirection + } + + case class Calc(date: DateLiteral, calc: CalcDirection, period: Period) extends Date +} diff --git a/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala b/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala new file mode 100644 index 00000000..769df244 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala @@ -0,0 +1,71 @@ +package docspell.query + +import cats._ +import cats.implicits._ + +import docspell.query.ItemQuery.Expr.AndExpr +import docspell.query.ItemQuery.Expr.NotExpr +import docspell.query.ItemQuery.Expr.OrExpr +import docspell.query.ItemQuery._ + +/** Currently, fulltext in a query is only supported when in "root + * AND" position + */ +object FulltextExtract { + + sealed trait Result + sealed trait SuccessResult extends Result + sealed trait FailureResult extends Result + object Result { + case class Success(query: Expr, fts: Option[String]) extends SuccessResult + case object TooMany extends FailureResult + case object UnsupportedPosition extends FailureResult + } + + def findFulltext(expr: Expr): Result = + lookForFulltext(expr) + + private def lookForFulltext(expr: Expr): Result = + expr match { + case Expr.Fulltext(ftq) => + Result.Success(ItemQuery.all.expr, ftq.some) + case Expr.AndExpr(inner) => + inner.collect({ case Expr.Fulltext(fq) => fq }) match { + case Nil => + checkPosition(expr, 0) + case e :: Nil => + val c = foldMap(isFulltextExpr)(expr) + if (c > 1) Result.TooMany + else Result.Success(expr, e.some) + case _ => + Result.TooMany + } + case _ => + checkPosition(expr, 0) + } + + private def checkPosition(expr: Expr, max: Int): Result = { + val c = foldMap(isFulltextExpr)(expr) + if (c > max) Result.UnsupportedPosition + else Result.Success(expr, None) + } + + private def foldMap[B: Monoid](f: Expr => B)(expr: Expr): B = + expr match { + case OrExpr(inner) => + inner.map(foldMap(f)).fold + case AndExpr(inner) => + inner.map(foldMap(f)).fold + case NotExpr(e) => + f(e) + case _ => + f(expr) + } + + private def isFulltextExpr(expr: Expr): Int = + expr match { + case Expr.Fulltext(_) => 1 + case _ => 0 + } + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala new file mode 100644 index 00000000..e2b7ef06 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala @@ -0,0 +1,183 @@ +package docspell.query + +import cats.data.{NonEmptyList => Nel} + +import docspell.query.ItemQuery.Attr.{DateAttr, IntAttr, StringAttr} + +/** A query evaluates to `true` or `false` given enough details about + * an item. + * + * It may consist of (field,op,value) tuples that specify some checks + * against a specific field of an item using some operator or a + * combination thereof. + */ +final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) { + def findFulltext: FulltextExtract.Result = + FulltextExtract.findFulltext(expr) +} + +object ItemQuery { + val all = ItemQuery(Expr.Exists(Attr.ItemId), Some("")) + + sealed trait Operator + object Operator { + case object Eq extends Operator + case object Neq extends Operator + case object Like extends Operator + case object Gt extends Operator + case object Lt extends Operator + case object Gte extends Operator + case object Lte extends Operator + } + + sealed trait TagOperator + object TagOperator { + case object AllMatch extends TagOperator + case object AnyMatch extends TagOperator + } + + sealed trait Attr + object Attr { + sealed trait StringAttr extends Attr + sealed trait DateAttr extends Attr + sealed trait IntAttr extends Attr + + case object ItemName extends StringAttr + case object ItemSource extends StringAttr + case object ItemNotes extends StringAttr + case object ItemId extends StringAttr + case object Date extends DateAttr + case object DueDate extends DateAttr + case object AttachCount extends IntAttr + + object Correspondent { + case object OrgId extends StringAttr + case object OrgName extends StringAttr + case object PersonId extends StringAttr + case object PersonName extends StringAttr + } + + object Concerning { + case object PersonId extends StringAttr + case object PersonName extends StringAttr + case object EquipId extends StringAttr + case object EquipName extends StringAttr + } + + object Folder { + case object FolderId extends StringAttr + case object FolderName extends StringAttr + } + } + + sealed trait Property + object Property { + final case class StringProperty(attr: StringAttr, value: String) extends Property + final case class DateProperty(attr: DateAttr, value: Date) extends Property + final case class IntProperty(attr: IntAttr, value: Int) extends Property + + def apply(sa: StringAttr, value: String): Property = + StringProperty(sa, value) + + def apply(da: DateAttr, value: Date): Property = + DateProperty(da, value) + + def apply(na: IntAttr, value: Int): Property = + IntProperty(na, value) + } + + sealed trait Expr { + def negate: Expr = + Expr.NotExpr(this) + } + + object Expr { + final case class AndExpr(expr: Nel[Expr]) extends Expr + final case class OrExpr(expr: Nel[Expr]) extends Expr + final case class NotExpr(expr: Expr) extends Expr { + override def negate: Expr = + expr + } + + final case class SimpleExpr(op: Operator, prop: Property) extends Expr + final case class Exists(field: Attr) extends Expr + final case class InExpr(attr: StringAttr, values: Nel[String]) extends Expr + final case class InDateExpr(attr: DateAttr, values: Nel[Date]) extends Expr + final case class InboxExpr(inbox: Boolean) extends Expr + final case class DirectionExpr(incoming: Boolean) extends Expr + + final case class TagIdsMatch(op: TagOperator, tags: Nel[String]) extends Expr + final case class TagsMatch(op: TagOperator, tags: Nel[String]) extends Expr + final case class TagCategoryMatch(op: TagOperator, cats: Nel[String]) extends Expr + + final case class CustomFieldMatch(name: String, op: Operator, value: String) + extends Expr + final case class CustomFieldIdMatch(id: String, op: Operator, value: String) + extends Expr + + final case class Fulltext(query: String) extends Expr + final case class ChecksumMatch(checksum: String) extends Expr + final case class AttachId(id: String) extends Expr + + // things that can be expressed with terms above + sealed trait MacroExpr extends Expr { + def body: Expr + } + final case class NamesMacro(searchTerm: String) extends MacroExpr { + val body = + Expr.or( + like(Attr.ItemName, searchTerm), + like(Attr.Correspondent.OrgName, searchTerm), + like(Attr.Correspondent.PersonName, searchTerm), + like(Attr.Concerning.PersonName, searchTerm), + like(Attr.Concerning.EquipName, searchTerm) + ) + } + + final case class CorrMacro(term: String) extends MacroExpr { + val body = + Expr.or( + like(Attr.Correspondent.OrgName, term), + like(Attr.Correspondent.PersonName, term) + ) + } + + final case class ConcMacro(term: String) extends MacroExpr { + val body = + Expr.or( + like(Attr.Concerning.PersonName, term), + like(Attr.Concerning.EquipName, term) + ) + } + + final case class DateRangeMacro(attr: DateAttr, left: Date, right: Date) + extends MacroExpr { + val body = + and(date(Operator.Gte, attr, left), date(Operator.Lt, attr, right)) + } + + final case class YearMacro(attr: DateAttr, year: Int) extends MacroExpr { + val body = + DateRangeMacro(attr, date(year), date(year + 1)) + + private def date(y: Int): Date = + Date(y, 1, 1).fold(throw _, identity) + } + + def or(expr0: Expr, exprs: Expr*): OrExpr = + OrExpr(Nel.of(expr0, exprs: _*)) + + def and(expr0: Expr, exprs: Expr*): AndExpr = + AndExpr(Nel.of(expr0, exprs: _*)) + + def string(op: Operator, attr: StringAttr, value: String): SimpleExpr = + SimpleExpr(op, Property(attr, value)) + + def like(attr: StringAttr, value: String): SimpleExpr = + string(Operator.Like, attr, value) + + def date(op: Operator, attr: DateAttr, value: Date): SimpleExpr = + SimpleExpr(op, Property(attr, value)) + } + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/ItemQueryParser.scala b/modules/query/shared/src/main/scala/docspell/query/ItemQueryParser.scala new file mode 100644 index 00000000..2c178140 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/ItemQueryParser.scala @@ -0,0 +1,21 @@ +package docspell.query + +import docspell.query.internal.ExprParser +import docspell.query.internal.ExprUtil + +object ItemQueryParser { + + def parse(input: String): Either[ParseFailure, ItemQuery] = + if (input.isEmpty) Right(ItemQuery.all) + else { + val in = if (input.charAt(0) == '(') input else s"(& $input )" + ExprParser + .parseQuery(in) + .left + .map(ParseFailure.fromError(in)) + .map(q => q.copy(expr = ExprUtil.reduce(q.expr))) + } + + def parseUnsafe(input: String): ItemQuery = + parse(input).fold(m => sys.error(m.render), identity) +} diff --git a/modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala b/modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala new file mode 100644 index 00000000..128a08c4 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala @@ -0,0 +1,105 @@ +package docspell.query + +import cats.data.{NonEmptyList => Nel} +import cats.parse.Parser +import cats.parse.Parser.Expectation.EndOfString +import cats.parse.Parser.Expectation.ExpectedFailureAt +import cats.parse.Parser.Expectation.Fail +import cats.parse.Parser.Expectation.FailWith +import cats.parse.Parser.Expectation.InRange +import cats.parse.Parser.Expectation.Length +import cats.parse.Parser.Expectation.OneOfStr +import cats.parse.Parser.Expectation.StartOfString + +final case class ParseFailure( + input: String, + failedAt: Int, + messages: Nel[ParseFailure.Message] +) { + + def render: String = { + val items = messages.map(_.render).toList.mkString(", ") + s"Failed to read input near $failedAt: $input\nDetails: $items" + } +} + +object ParseFailure { + + sealed trait Message { + def offset: Int + def render: String + } + final case class SimpleMessage(offset: Int, msg: String) extends Message { + def render: String = + s"Failed at $offset: $msg" + } + final case class ExpectMessage(offset: Int, expected: List[String], exhaustive: Boolean) + extends Message { + def render: String = { + val opts = expected.mkString(", ") + val dots = if (exhaustive) "" else "…" + s"Expected: ${opts}${dots}" + } + } + + private[query] def fromError(input: String)(pe: Parser.Error): ParseFailure = + ParseFailure( + input, + pe.failedAtOffset, + packMsg(Parser.Expectation.unify(pe.expected).map(expectationToMsg)) + ) + + private[query] def packMsg(msg: Nel[Message]): Nel[Message] = { + val expectMsg = combineExpected(msg.collect({ case em: ExpectMessage => em })) + .sortBy(_.offset) + .headOption + + val simpleMsg = msg.collect({ case sm: SimpleMessage => sm }) + + Nel.fromListUnsafe((simpleMsg ++ expectMsg).sortBy(_.offset)) + } + + private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] = + msg + .groupBy(_.offset) + .map({ case (offset, es) => + ExpectMessage( + offset, + es.flatMap(_.expected).distinct.sorted, + es.forall(_.exhaustive) + ) + }) + .toList + + private[query] def expectationToMsg(e: Parser.Expectation): Message = + e match { + case StartOfString(offset) => + SimpleMessage(offset, "Expected start of string") + + case FailWith(offset, message) => + SimpleMessage(offset, message) + + case InRange(offset, lower, upper) => + if (lower == upper) ExpectMessage(offset, List(lower.toString), true) + else { + val expect = s"${lower}-${upper}" + ExpectMessage(offset, List(expect), true) + } + + case Length(offset, expected, actual) => + SimpleMessage(offset, s"Expected input of length $expected, but got $actual") + + case ExpectedFailureAt(offset, matched) => + SimpleMessage(offset, s"Expected failing, but matched '$matched'") + + case EndOfString(offset, length) => + SimpleMessage(offset, s"Expected end of string at length: $length") + + case Fail(offset) => + SimpleMessage(offset, s"Failed to parse near $offset") + + case OneOfStr(offset, strs) => + val options = strs.take(8) + ExpectMessage(offset, options.take(7), options.size < 8) + } +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/AttrParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/AttrParser.scala new file mode 100644 index 00000000..6d74ea59 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/AttrParser.scala @@ -0,0 +1,99 @@ +package docspell.query.internal + +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery.Attr +import docspell.query.internal.{Constants => C} + +object AttrParser { + + val name: P[Attr.StringAttr] = + P.ignoreCase(C.name).as(Attr.ItemName) + + val source: P[Attr.StringAttr] = + P.ignoreCase(C.source).as(Attr.ItemSource) + + val id: P[Attr.StringAttr] = + P.ignoreCase(C.id).as(Attr.ItemId) + + val date: P[Attr.DateAttr] = + P.ignoreCase(C.date).as(Attr.Date) + + val notes: P[Attr.StringAttr] = + P.ignoreCase(C.notes).as(Attr.ItemNotes) + + val dueDate: P[Attr.DateAttr] = + P.ignoreCase(C.due).as(Attr.DueDate) + + val corrOrgId: P[Attr.StringAttr] = + P.ignoreCase(C.corrOrgId) + .as(Attr.Correspondent.OrgId) + + val corrOrgName: P[Attr.StringAttr] = + P.ignoreCase(C.corrOrgName) + .as(Attr.Correspondent.OrgName) + + val corrPersId: P[Attr.StringAttr] = + P.ignoreCase(C.corrPersId) + .as(Attr.Correspondent.PersonId) + + val corrPersName: P[Attr.StringAttr] = + P.ignoreCase(C.corrPersName) + .as(Attr.Correspondent.PersonName) + + val concPersId: P[Attr.StringAttr] = + P.ignoreCase(C.concPersId) + .as(Attr.Concerning.PersonId) + + val concPersName: P[Attr.StringAttr] = + P.ignoreCase(C.concPersName) + .as(Attr.Concerning.PersonName) + + val concEquipId: P[Attr.StringAttr] = + P.ignoreCase(C.concEquipId) + .as(Attr.Concerning.EquipId) + + val concEquipName: P[Attr.StringAttr] = + P.ignoreCase(C.concEquipName) + .as(Attr.Concerning.EquipName) + + val folderId: P[Attr.StringAttr] = + P.ignoreCase(C.folderId).as(Attr.Folder.FolderId) + + val folderName: P[Attr.StringAttr] = + P.ignoreCase(C.folder).as(Attr.Folder.FolderName) + + val attachCountAttr: P[Attr.IntAttr] = + P.ignoreCase(C.attachCount).as(Attr.AttachCount) + + // combining grouped by type + + val intAttr: P[Attr.IntAttr] = + attachCountAttr + + val dateAttr: P[Attr.DateAttr] = + P.oneOf(List(date, dueDate)) + + val stringAttr: P[Attr.StringAttr] = + P.oneOf( + List( + name, + source, + id, + notes, + corrOrgId, + corrOrgName, + corrPersId, + corrPersName, + concPersId, + concPersName, + concEquipId, + concEquipName, + folderId, + folderName + ) + ) + + val anyAttr: P[Attr] = + P.oneOf(List(dateAttr, stringAttr, intAttr)) +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/BasicParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/BasicParser.scala new file mode 100644 index 00000000..bb38fc90 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/BasicParser.scala @@ -0,0 +1,47 @@ +package docspell.query.internal + +import cats.data.{NonEmptyList => Nel} +import cats.parse.{Parser => P, Parser0} + +object BasicParser { + private[this] val whitespace: P[Unit] = P.charIn(" \t\r\n").void + + val ws0: Parser0[Unit] = whitespace.rep0.void + val ws1: P[Unit] = whitespace.rep.void + + val stringListSep: P[Unit] = + (ws0.with1.soft ~ P.char(',') ~ ws0).void + + private[this] val basicString: P[String] = + P.charsWhile(c => + c > ' ' && c != '"' && c != '\\' && c != ',' && c != '[' && c != ']' && c != '(' && c != ')' + ) + + private[this] val identChars: Set[Char] = + (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "-_.").toSet + + val parenAnd: P[Unit] = + P.stringIn(List("(&", "(and")).void <* ws0 + + val parenClose: P[Unit] = + ws0.soft.with1 *> P.char(')') + + val parenOr: P[Unit] = + P.stringIn(List("(|", "(or")).void <* ws0 + + val identParser: P[String] = + P.charsWhile(identChars.contains) + + val singleString: P[String] = + basicString.backtrack.orElse(StringUtil.quoted('"')) + + val stringOrMore: P[Nel[String]] = + singleString.repSep(stringListSep) + + val bool: P[Boolean] = { + val trueP = P.stringIn(List("yes", "true", "Yes", "True")).as(true) + val falseP = P.stringIn(List("no", "false", "No", "False")).as(false) + trueP | falseP + } + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/Constants.scala b/modules/query/shared/src/main/scala/docspell/query/internal/Constants.scala new file mode 100644 index 00000000..232d6940 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/Constants.scala @@ -0,0 +1,50 @@ +package docspell.query.internal + +object Constants { + + val attachCount = "attach.count" + val attachId = "attach.id" + val cat = "cat" + val checksum = "checksum" + val conc = "conc" + val concEquipId = "conc.equip.id" + val concEquipName = "conc.equip.name" + val concPersId = "conc.pers.id" + val concPersName = "conc.pers.name" + val content = "content" + val corr = "corr" + val corrOrgId = "corr.org.id" + val corrOrgName = "corr.org.name" + val corrPersId = "corr.pers.id" + val corrPersName = "corr.pers.name" + val customField = "f" + val customFieldId = "f.id" + val date = "date" + val dateIn = "dateIn" + val due = "due" + val dueIn = "dueIn" + val exist = "exist" + val folder = "folder" + val folderId = "folder.id" + val id = "id" + val inbox = "inbox" + val incoming = "incoming" + val name = "name" + val names = "names" + val notPrefix = '!' + val notes = "notes" + val source = "source" + val tag = "tag" + val tagId = "tag.id" + val year = "year" + + // operators + val eqs = '=' + val gt = '>' + val gte = ">=" + val in = "~=" + val like = ':' + val lt = '<' + val lte = "<=" + val neq = "!=" +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/DateParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/DateParser.scala new file mode 100644 index 00000000..02d99926 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/DateParser.scala @@ -0,0 +1,108 @@ +package docspell.query.internal + +import java.time.Period + +import cats.data.{NonEmptyList => Nel} +import cats.parse.{Numbers, Parser => P} + +import docspell.query.Date + +object DateParser { + private[this] val longParser: P[Long] = + Numbers.bigInt.map(_.longValue) + + private[this] val digits4: P[Int] = + Numbers.digit + .repExactlyAs[String](4) + .map(_.toInt) + private[this] val digits2: P[Int] = + Numbers.digit + .repExactlyAs[String](2) + .map(_.toInt) + + private[this] val month: P[Int] = + digits2.filter(n => n >= 1 && n <= 12) + + private[this] val day: P[Int] = + digits2.filter(n => n >= 1 && n <= 31) + + private val dateSep: P[Unit] = + P.charIn('-', '/').void + + private val dateString: P[((Int, Option[Int]), Option[Int])] = + digits4 ~ (dateSep *> month).? ~ (dateSep *> day).? + + private[internal] val dateFromString: P[Date.DateLiteral] = + dateString.mapFilter { case ((year, month), day) => + Date(year, month.getOrElse(1), day.getOrElse(1)).toOption + } + + private[internal] val dateFromMillis: P[Date.DateLiteral] = + P.string("ms") *> longParser.map(Date.apply) + + private val dateFromToday: P[Date.DateLiteral] = + P.string("today").as(Date.Today) + + val yearOnly: P[Int] = + digits4 + + val dateLiteral: P[Date.DateLiteral] = + P.oneOf(List(dateFromString, dateFromToday, dateFromMillis)) + + // val dateLiteralOrMore: P[NonEmptyList[Date.DateLiteral]] = + // dateLiteral.repSep(BasicParser.stringListSep) + + val dateCalcDirection: P[Date.CalcDirection] = + P.oneOf( + List( + P.char('+').as(Date.CalcDirection.Plus), + P.char('-').as(Date.CalcDirection.Minus) + ) + ) + + def periodPart(unitSuffix: Char, f: Int => Period): P[Period] = + ((Numbers.nonZeroDigit ~ Numbers.digits0).void.string.soft <* P.ignoreCaseChar( + unitSuffix + )) + .map(n => f(n.toInt)) + + private[this] val periodMonths: P[Period] = + periodPart('m', n => Period.ofMonths(n)) + + private[this] val periodDays: P[Period] = + periodPart('d', n => Period.ofDays(n)) + + val period: P[Period] = + periodDays.eitherOr(periodMonths).map(_.fold(identity, identity)) + + val periods: P[Period] = + period.rep.map(_.reduceLeft((p0, p1) => p0.plus(p1))) + + val dateRange: P[(Date, Date)] = + ((dateLiteral <* P.char(';')) ~ dateCalcDirection.eitherOr(P.char('/')) ~ period) + .map { case ((date, calc), period) => + calc match { + case Right(Date.CalcDirection.Plus) => + (date, Date.Calc(date, Date.CalcDirection.Plus, period)) + case Right(Date.CalcDirection.Minus) => + (Date.Calc(date, Date.CalcDirection.Minus, period), date) + case Left(_) => + ( + Date.Calc(date, Date.CalcDirection.Minus, period), + Date.Calc(date, Date.CalcDirection.Plus, period) + ) + } + } + + val date: P[Date] = + (dateLiteral ~ (P.char(';') *> dateCalcDirection ~ period).?).map { + case (date, Some((c, p))) => + Date.Calc(date, c, p) + + case (date, None) => + date + } + + val dateOrMore: P[Nel[Date]] = + date.repSep(BasicParser.stringListSep) +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/ExprParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/ExprParser.scala new file mode 100644 index 00000000..de6e9165 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/ExprParser.scala @@ -0,0 +1,39 @@ +package docspell.query.internal + +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery +import docspell.query.ItemQuery._ +import docspell.query.internal.{Constants => C} + +object ExprParser { + + def and(inner: P[Expr]): P[Expr.AndExpr] = + inner + .repSep(BasicParser.ws1) + .between(BasicParser.parenAnd, BasicParser.parenClose) + .map(Expr.AndExpr.apply) + + def or(inner: P[Expr]): P[Expr.OrExpr] = + inner + .repSep(BasicParser.ws1) + .between(BasicParser.parenOr, BasicParser.parenClose) + .map(Expr.OrExpr.apply) + + def not(inner: P[Expr]): P[Expr] = + (P.char(C.notPrefix) *> inner).map(_.negate) + + val exprParser: P[Expr] = + P.recursive[Expr] { recurse => + val andP = and(recurse) + val orP = or(recurse) + val notP = not(recurse) + val macros = MacroParser.all + P.oneOf(macros :: SimpleExprParser.simpleExpr :: andP :: orP :: notP :: Nil) + } + + def parseQuery(input: String): Either[P.Error, ItemQuery] = { + val p = BasicParser.ws0 *> exprParser <* (BasicParser.ws0 ~ P.end) + p.parseAll(input).map(expr => ItemQuery(expr, Some(input))) + } +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala b/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala new file mode 100644 index 00000000..df81983f --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala @@ -0,0 +1,88 @@ +package docspell.query.internal + +import cats.data.{NonEmptyList => Nel} + +import docspell.query.ItemQuery.Expr._ +import docspell.query.ItemQuery._ + +object ExprUtil { + + /** Does some basic transformation, like unfolding nested and trees + * containing one value etc. + */ + def reduce(expr: Expr): Expr = + expr match { + case AndExpr(inner) => + val nodes = spliceAnd(inner) + if (nodes.tail.isEmpty) reduce(nodes.head) + else AndExpr(nodes.map(reduce)) + + case OrExpr(inner) => + val nodes = spliceOr(inner) + if (nodes.tail.isEmpty) reduce(nodes.head) + else OrExpr(nodes.map(reduce)) + + case NotExpr(inner) => + inner match { + case NotExpr(inner2) => + reduce(inner2) + case InboxExpr(flag) => + InboxExpr(!flag) + case DirectionExpr(flag) => + DirectionExpr(!flag) + case _ => + expr + } + + case m: MacroExpr => + reduce(m.body) + + case DirectionExpr(_) => + expr + + case InboxExpr(_) => + expr + + case InExpr(_, _) => + expr + + case InDateExpr(_, _) => + expr + + case TagsMatch(_, _) => + expr + case TagIdsMatch(_, _) => + expr + case Exists(_) => + expr + case Fulltext(_) => + expr + case SimpleExpr(_, _) => + expr + case TagCategoryMatch(_, _) => + expr + case CustomFieldMatch(_, _, _) => + expr + case CustomFieldIdMatch(_, _, _) => + expr + case ChecksumMatch(_) => + expr + case AttachId(_) => + expr + } + + private def spliceAnd(nodes: Nel[Expr]): Nel[Expr] = + nodes.flatMap { + case Expr.AndExpr(inner) => + spliceAnd(inner) + case node => + Nel.of(node) + } + private def spliceOr(nodes: Nel[Expr]): Nel[Expr] = + nodes.flatMap { + case Expr.OrExpr(inner) => + spliceOr(inner) + case node => + Nel.of(node) + } +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/MacroParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/MacroParser.scala new file mode 100644 index 00000000..44b2d93f --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/MacroParser.scala @@ -0,0 +1,55 @@ +package docspell.query.internal + +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery._ +import docspell.query.internal.{Constants => C} + +object MacroParser { + private def macroDef(name: String): P[Unit] = + P.ignoreCase(name).soft.with1 <* P.char(':') + + private def dateRangeMacroImpl( + name: String, + attr: Attr.DateAttr + ): P[Expr.DateRangeMacro] = + (macroDef(name) *> DateParser.dateRange).map { case (left, right) => + Expr.DateRangeMacro(attr, left, right) + } + + private def yearMacroImpl(name: String, attr: Attr.DateAttr): P[Expr.YearMacro] = + (macroDef(name) *> DateParser.yearOnly).map(year => Expr.YearMacro(attr, year)) + + val namesMacro: P[Expr.NamesMacro] = + (macroDef(C.names) *> BasicParser.singleString).map(Expr.NamesMacro.apply) + + val dateRangeMacro: P[Expr.DateRangeMacro] = + dateRangeMacroImpl(C.dateIn, Attr.Date) + + val dueDateRangeMacro: P[Expr.DateRangeMacro] = + dateRangeMacroImpl(C.dueIn, Attr.DueDate) + + val yearDateMacro: P[Expr.YearMacro] = + yearMacroImpl(C.year, Attr.Date) + + val corrMacro: P[Expr.CorrMacro] = + (macroDef(C.corr) *> BasicParser.singleString).map(Expr.CorrMacro.apply) + + val concMacro: P[Expr.ConcMacro] = + (macroDef(C.conc) *> BasicParser.singleString).map(Expr.ConcMacro.apply) + + // --- all macro parser + + val all: P[Expr] = + P.oneOf( + List( + namesMacro, + dateRangeMacro, + dueDateRangeMacro, + yearDateMacro, + corrMacro, + concMacro + ) + ) + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/OperatorParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/OperatorParser.scala new file mode 100644 index 00000000..2b5956a4 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/OperatorParser.scala @@ -0,0 +1,41 @@ +package docspell.query.internal + +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery._ +import docspell.query.internal.{Constants => C} + +object OperatorParser { + private[this] val Eq: P[Operator] = + P.char(C.eqs).as(Operator.Eq) + + private[this] val Neq: P[Operator] = + P.string(C.neq).as(Operator.Neq) + + private[this] val Like: P[Operator] = + P.char(C.like).as(Operator.Like) + + private[this] val Gt: P[Operator] = + P.char(C.gt).as(Operator.Gt) + + private[this] val Lt: P[Operator] = + P.char(C.lt).as(Operator.Lt) + + private[this] val Gte: P[Operator] = + P.string(C.gte).as(Operator.Gte) + + private[this] val Lte: P[Operator] = + P.string(C.lte).as(Operator.Lte) + + val op: P[Operator] = + P.oneOf(List(Like, Eq, Neq, Gte, Lte, Gt, Lt)) + + private[this] val anyOp: P[TagOperator] = + P.char(C.like).as(TagOperator.AnyMatch) + + private[this] val allOp: P[TagOperator] = + P.char(C.eqs).as(TagOperator.AllMatch) + + val tagOp: P[TagOperator] = + P.oneOf(List(anyOp, allOp)) +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/SimpleExprParser.scala b/modules/query/shared/src/main/scala/docspell/query/internal/SimpleExprParser.scala new file mode 100644 index 00000000..950df0cb --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/SimpleExprParser.scala @@ -0,0 +1,125 @@ +package docspell.query.internal + +import cats.parse.Numbers +import cats.parse.{Parser => P} + +import docspell.query.ItemQuery._ +import docspell.query.internal.{Constants => C} + +object SimpleExprParser { + + private[this] val op: P[Operator] = + OperatorParser.op.surroundedBy(BasicParser.ws0) + + private[this] val inOp: P[Unit] = + P.string(C.in).surroundedBy(BasicParser.ws0) + + private[this] val inOrOpStr = + P.eitherOr(op ~ BasicParser.singleString, inOp *> BasicParser.stringOrMore) + + private[this] val inOrOpDate = + P.eitherOr(op ~ DateParser.date, inOp *> DateParser.dateOrMore) + + private[this] val opInt = + op ~ Numbers.digits.map(_.toInt) + + val stringExpr: P[Expr] = + (AttrParser.stringAttr ~ inOrOpStr).map { + case (attr, Right((op, value))) => + Expr.SimpleExpr(op, Property.StringProperty(attr, value)) + case (attr, Left(values)) => + Expr.InExpr(attr, values) + } + + val dateExpr: P[Expr] = + (AttrParser.dateAttr ~ inOrOpDate).map { + case (attr, Right((op, value))) => + Expr.SimpleExpr(op, Property.DateProperty(attr, value)) + case (attr, Left(values)) => + Expr.InDateExpr(attr, values) + } + + val intExpr: P[Expr] = + (AttrParser.intAttr ~ opInt).map { case (attr, (op, value)) => + Expr.SimpleExpr(op, Property(attr, value)) + } + + val existsExpr: P[Expr.Exists] = + (P.ignoreCase(C.exist) *> P.char(C.like) *> AttrParser.anyAttr).map(attr => + Expr.Exists(attr) + ) + + val fulltextExpr: P[Expr.Fulltext] = + (P.ignoreCase(C.content) *> P.char(C.like) *> BasicParser.singleString).map(q => + Expr.Fulltext(q) + ) + + val tagIdExpr: P[Expr.TagIdsMatch] = + (P.ignoreCase(C.tagId) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map { + case (op, values) => + Expr.TagIdsMatch(op, values) + } + + val tagExpr: P[Expr.TagsMatch] = + (P.ignoreCase(C.tag) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map { + case (op, values) => + Expr.TagsMatch(op, values) + } + + val catExpr: P[Expr.TagCategoryMatch] = + (P.ignoreCase(C.cat) *> OperatorParser.tagOp ~ BasicParser.stringOrMore).map { + case (op, values) => + Expr.TagCategoryMatch(op, values) + } + + val customFieldExpr: P[Expr.CustomFieldMatch] = + (P.string(C.customField) *> P.char( + C.like + ) *> BasicParser.identParser ~ op ~ BasicParser.singleString) + .map { case ((name, op), value) => + Expr.CustomFieldMatch(name, op, value) + } + + val customFieldIdExpr: P[Expr.CustomFieldIdMatch] = + (P.string(C.customFieldId) *> P.char( + C.like + ) *> BasicParser.identParser ~ op ~ BasicParser.singleString) + .map { case ((name, op), value) => + Expr.CustomFieldIdMatch(name, op, value) + } + + val inboxExpr: P[Expr.InboxExpr] = + (P.string(C.inbox) *> P.char(C.like) *> BasicParser.bool).map(Expr.InboxExpr.apply) + + val dirExpr: P[Expr.DirectionExpr] = + (P.string(C.incoming) *> P.char(C.like) *> BasicParser.bool) + .map(Expr.DirectionExpr.apply) + + val checksumExpr: P[Expr.ChecksumMatch] = + (P.string(C.checksum) *> P.char(C.like) *> BasicParser.singleString) + .map(Expr.ChecksumMatch.apply) + + val attachIdExpr: P[Expr.AttachId] = + (P.ignoreCase(C.attachId) *> P.char(C.eqs) *> BasicParser.singleString) + .map(Expr.AttachId.apply) + + val simpleExpr: P[Expr] = + P.oneOf( + List( + dateExpr, + stringExpr, + intExpr, + existsExpr, + fulltextExpr, + tagIdExpr, + tagExpr, + catExpr, + customFieldIdExpr, + customFieldExpr, + inboxExpr, + dirExpr, + checksumExpr, + attachIdExpr + ) + ) +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/StringUtil.scala b/modules/query/shared/src/main/scala/docspell/query/internal/StringUtil.scala new file mode 100644 index 00000000..fb81ce14 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/internal/StringUtil.scala @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2021 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package docspell.query.internal + +// modified, from +// https://github.com/typelevel/cats-parse/blob/e7a58ef15925358fbe7a4c0c1a204296e366a06c/bench/src/main/scala/cats/parse/bench/self.scala +import cats.parse.{Parser => P, Parser0 => P0} + +object StringUtil { + + def quoted(q: Char): P[String] = + Util.escapedString(q) + + private object Util extends GenericStringUtil { + lazy val decodeTable: Map[Char, Char] = + Map( + ('\\', '\\'), + ('\'', '\''), + ('\"', '\"'), + ('n', '\n'), + ('r', '\r'), + ('t', '\t') + ) + } + abstract private class GenericStringUtil { + protected def decodeTable: Map[Char, Char] + + private val encodeTable = decodeTable.iterator.map { case (v, k) => + (k, s"\\$v") + }.toMap + + private val nonPrintEscape: Array[String] = + (0 until 32).map { c => + val strHex = c.toHexString + val strPad = List.fill(4 - strHex.length)('0').mkString + s"\\u$strPad$strHex" + }.toArray + + val escapedToken: P[Unit] = { + val escapes = P.charIn(decodeTable.keys.toSeq) + + val oct = P.charIn('0' to '7') + val octP = P.char('o') ~ oct ~ oct + + val hex = P.charIn(('0' to '9') ++ ('a' to 'f') ++ ('A' to 'F')) + val hex2 = hex ~ hex + val hexP = P.char('x') ~ hex2 + + val hex4 = hex2 ~ hex2 + val u4 = P.char('u') ~ hex4 + val hex8 = hex4 ~ hex4 + val u8 = P.char('U') ~ hex8 + + val after = P.oneOf[Any](escapes :: octP :: hexP :: u4 :: u8 :: Nil) + (P.char('\\') ~ after).void + } + + /** String content without the delimiter + */ + def undelimitedString(endP: P[Unit]): P[String] = + escapedToken.backtrack + .orElse((!endP).with1 ~ P.anyChar) + .rep + .string + .flatMap { str => + unescape(str) match { + case Right(str1) => P.pure(str1) + case Left(_) => P.fail + } + } + + private val simpleString: P0[String] = + P.charsWhile0(c => c >= ' ' && c != '"' && c != '\\') + + def escapedString(q: Char): P[String] = { + val end: P[Unit] = P.char(q) + end *> ((simpleString <* end).backtrack + .orElse(undelimitedString(end) <* end)) + } + + def escape(quoteChar: Char, str: String): String = { + // We can ignore escaping the opposite character used for the string + // x isn't escaped anyway and is kind of a hack here + val ignoreEscape = + if (quoteChar == '\'') '"' else if (quoteChar == '"') '\'' else 'x' + str.flatMap { c => + if (c == ignoreEscape) c.toString + else + encodeTable.get(c) match { + case None => + if (c < ' ') nonPrintEscape(c.toInt) + else c.toString + case Some(esc) => esc + } + } + } + + def unescape(str: String): Either[Int, String] = { + val sb = new java.lang.StringBuilder + def decodeNum(idx: Int, size: Int, base: Int): Int = { + val end = idx + size + if (end <= str.length) { + val intStr = str.substring(idx, end) + val asInt = + try Integer.parseInt(intStr, base) + catch { case _: NumberFormatException => ~idx } + sb.append(asInt.toChar) + end + } else ~(str.length) + } + @annotation.tailrec + def loop(idx: Int): Int = + if (idx >= str.length) { + // done + idx + } else if (idx < 0) { + // error from decodeNum + idx + } else { + val c0 = str.charAt(idx) + if (c0 != '\\') { + sb.append(c0) + loop(idx + 1) + } else { + // str(idx) == \ + val nextIdx = idx + 1 + if (nextIdx >= str.length) { + // error we expect there to be a character after \ + ~idx + } else { + val c = str.charAt(nextIdx) + decodeTable.get(c) match { + case Some(d) => + sb.append(d) + loop(idx + 2) + case None => + c match { + case 'o' => loop(decodeNum(idx + 2, 2, 8)) + case 'x' => loop(decodeNum(idx + 2, 2, 16)) + case 'u' => loop(decodeNum(idx + 2, 4, 16)) + case 'U' => loop(decodeNum(idx + 2, 8, 16)) + case other => + // \c is interpreted as just \c, if the character isn't escaped + sb.append('\\') + sb.append(other) + loop(idx + 2) + } + } + } + } + } + + val res = loop(0) + if (res < 0) Left(~res) + else Right(sb.toString) + } + } + +} diff --git a/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala b/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala new file mode 100644 index 00000000..755ea121 --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala @@ -0,0 +1,62 @@ +package docspell.query + +import cats.implicits._ +import munit._ +import docspell.query.FulltextExtract.Result + +class FulltextExtractTest extends FunSuite { + + def findFts(q: String): Result = { + val p = ItemQueryParser.parseUnsafe(q) + FulltextExtract.findFulltext(p.expr) + } + + def assertFts(qstr: String, expect: Result) = + assertEquals(findFts(qstr), expect) + + def assertFtsSuccess(qstr: String, expect: Option[String]) = { + val q = ItemQueryParser.parseUnsafe(qstr) + assertEquals(findFts(qstr), Result.Success(q.expr, expect)) + } + + test("find fulltext as root") { + assertEquals(findFts("content:what"), Result.Success(ItemQuery.all.expr, "what".some)) + assertEquals( + findFts("content:\"what hello\""), + Result.Success(ItemQuery.all.expr, "what hello".some) + ) + assertEquals( + findFts("content:\"what OR hello\""), + Result.Success(ItemQuery.all.expr, "what OR hello".some) + ) + + assertEquals( + findFts("(& content:\"what OR hello\" )"), + Result.Success(ItemQuery.all.expr, "what OR hello".some) + ) + } + + test("find no fulltext") { + assertFtsSuccess("name:test", None) + } + + test("find fulltext within and") { + assertFtsSuccess("content:what name:test", "what".some) + assertFtsSuccess("names:marc* content:what name:test", "what".some) + assertFtsSuccess( + "names:marc* date:2021-02 content:\"what else\" name:test", + "what else".some + ) + } + + test("too many fulltext searches") { + assertFts("content:yes content:no", Result.TooMany) + assertFts("content:yes (| name:test content:no)", Result.TooMany) + assertFts("content:yes (| name:test (& date:2021-02 content:no))", Result.TooMany) + } + + test("wrong fulltext search position") { + assertFts("name:test (| date:2021-02 content:yes)", Result.UnsupportedPosition) + assertFtsSuccess("name:test (& date:2021-02 content:yes)", "yes".some) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/AttrParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/AttrParserTest.scala new file mode 100644 index 00000000..34c2270f --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/AttrParserTest.scala @@ -0,0 +1,37 @@ +package docspell.query.internal + +import docspell.query.ItemQuery.Attr +import docspell.query.internal.AttrParser +import munit._ + +class AttrParserTest extends FunSuite { + + test("string attributes") { + val p = AttrParser.stringAttr + assertEquals(p.parseAll("name"), Right(Attr.ItemName)) + assertEquals(p.parseAll("source"), Right(Attr.ItemSource)) + assertEquals(p.parseAll("id"), Right(Attr.ItemId)) + assertEquals(p.parseAll("corr.org.id"), Right(Attr.Correspondent.OrgId)) + assertEquals(p.parseAll("corr.org.name"), Right(Attr.Correspondent.OrgName)) + assertEquals(p.parseAll("conc.pers.id"), Right(Attr.Concerning.PersonId)) + assertEquals(p.parseAll("conc.pers.name"), Right(Attr.Concerning.PersonName)) + assertEquals(p.parseAll("folder"), Right(Attr.Folder.FolderName)) + assertEquals(p.parseAll("folder.id"), Right(Attr.Folder.FolderId)) + } + + test("date attributes") { + val p = AttrParser.dateAttr + assertEquals(p.parseAll("date"), Right(Attr.Date)) + assertEquals(p.parseAll("due"), Right(Attr.DueDate)) + } + + test("all attributes parser") { + val p = AttrParser.anyAttr + assertEquals(p.parseAll("date"), Right(Attr.Date)) + assertEquals(p.parseAll("name"), Right(Attr.ItemName)) + assertEquals(p.parseAll("source"), Right(Attr.ItemSource)) + assertEquals(p.parseAll("id"), Right(Attr.ItemId)) + assertEquals(p.parseAll("corr.org.id"), Right(Attr.Correspondent.OrgId)) + assertEquals(p.parseAll("corr.org.name"), Right(Attr.Correspondent.OrgName)) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/BasicParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/BasicParserTest.scala new file mode 100644 index 00000000..e397ce9b --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/BasicParserTest.scala @@ -0,0 +1,30 @@ +package docspell.query.internal + +import munit._ +import cats.data.{NonEmptyList => Nel} +import docspell.query.internal.BasicParser + +class BasicParserTest extends FunSuite { + test("single string values") { + val p = BasicParser.singleString + assertEquals(p.parseAll("abcde"), Right("abcde")) + assert(p.parseAll("ab cd").isLeft) + assertEquals(p.parseAll(""""ab cd""""), Right("ab cd")) + assertEquals(p.parseAll(""""and \"this\" is""""), Right("""and "this" is""")) + } + + test("string list values") { + val p = BasicParser.stringOrMore + assertEquals(p.parseAll("ab,cd,123"), Right(Nel.of("ab", "cd", "123"))) + assertEquals(p.parseAll("a,b"), Right(Nel.of("a", "b"))) + assert(p.parseAll("[a,b").isLeft) + } + + test("stringvalue") { + val p = BasicParser.stringOrMore + assertEquals(p.parseAll("abcde"), Right(Nel.of("abcde"))) + assertEquals(p.parseAll(""""a,b,c""""), Right(Nel.of("a,b,c"))) + + assertEquals(p.parse("a, b, c "), Right((" ", Nel.of("a", "b", "c")))) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/DateParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/DateParserTest.scala new file mode 100644 index 00000000..ec836ced --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/DateParserTest.scala @@ -0,0 +1,60 @@ +package docspell.query.internal + +import munit._ +import docspell.query.Date +import java.time.Period + +class DateParserTest extends FunSuite with ValueHelper { + + test("local date string") { + val p = DateParser.dateFromString + assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22))) + assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11))) + assertEquals(p.parseAll("2032-01-21"), Right(ld(2032, 1, 21))) + assert(p.parseAll("0-0-0").isLeft) + assert(p.parseAll("2021-02-30").isLeft) + } + + test("local date millis") { + val p = DateParser.dateFromMillis + assertEquals(p.parseAll("ms0"), Right(Date(0))) + assertEquals( + p.parseAll("ms1600000065463"), + Right(Date(1600000065463L)) + ) + } + + test("local date") { + val p = DateParser.date + assertEquals(p.parseAll("2021-02-22"), Right(ld(2021, 2, 22))) + assertEquals(p.parseAll("1999-11-11"), Right(ld(1999, 11, 11))) + assertEquals(p.parseAll("ms0"), Right(Date(0))) + assertEquals(p.parseAll("ms1600000065463"), Right(Date(1600000065463L))) + } + + test("local partial date") { + val p = DateParser.date + assertEquals(p.parseAll("2021-04"), Right(ld(2021, 4, 1))) + assertEquals(p.parseAll("2021-12"), Right(ld(2021, 12, 1))) + assert(p.parseAll("2021-13").isLeft) + assert(p.parseAll("2021-28").isLeft) + assertEquals(p.parseAll("2021"), Right(ld(2021, 1, 1))) + } + + test("date calcs") { + val p = DateParser.date + assertEquals(p.parseAll("2020-02;+2d"), Right(ldPlus(2020, 2, 1, Period.ofDays(2)))) + assertEquals( + p.parseAll("today;-2m"), + Right(Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2))) + ) + } + + test("period") { + val p = DateParser.periods + assertEquals(p.parseAll("15d"), Right(Period.ofDays(15))) + assertEquals(p.parseAll("15m"), Right(Period.ofMonths(15))) + assertEquals(p.parseAll("15d10m"), Right(Period.ofMonths(10).plus(Period.ofDays(15)))) + assertEquals(p.parseAll("10m15d"), Right(Period.ofMonths(10).plus(Period.ofDays(15)))) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/ExprParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/ExprParserTest.scala new file mode 100644 index 00000000..07e14e9b --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/ExprParserTest.scala @@ -0,0 +1,92 @@ +package docspell.query.internal + +import docspell.query.ItemQuery._ +import munit._ +import cats.data.{NonEmptyList => Nel} + +class ExprParserTest extends FunSuite with ValueHelper { + + test("simple expr") { + val p = ExprParser.exprParser + assertEquals( + p.parseAll("name:hello"), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello")) + ) + } + + test("and") { + val p = ExprParser.exprParser + assertEquals( + p.parseAll("(& name:hello source=webapp )"), + Right( + Expr.AndExpr( + Nel.of( + stringExpr(Operator.Like, Attr.ItemName, "hello"), + stringExpr(Operator.Eq, Attr.ItemSource, "webapp") + ) + ) + ) + ) + } + + test("or") { + val p = ExprParser.exprParser + assertEquals( + p.parseAll("(| name:hello source=webapp )"), + Right( + Expr.OrExpr( + Nel.of( + stringExpr(Operator.Like, Attr.ItemName, "hello"), + stringExpr(Operator.Eq, Attr.ItemSource, "webapp") + ) + ) + ) + ) + } + + test("tag list inside and/or") { + val p = ExprParser.exprParser + assertEquals( + p.parseAll("(& tag:a,b,c)"), + Right( + Expr.AndExpr( + Nel.of( + Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c")) + ) + ) + ) + ) + assertEquals( + p.parseAll("(& tag:a,b,c )"), + Right( + Expr.AndExpr( + Nel.of( + Expr.TagsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c")) + ) + ) + ) + ) + } + + test("nest and/ with simple expr") { + val p = ExprParser.exprParser + assertEquals( + p.parseAll("(& (& f:usd=\"4.99\" ) source:*test* )"), + Right( + Expr.and( + Expr.and(Expr.CustomFieldMatch("usd", Operator.Eq, "4.99")), + Expr.string(Operator.Like, Attr.ItemSource, "*test*") + ) + ) + ) + assertEquals( + p.parseAll("(& (& f:usd=\"4.99\" ) (| source:*test*) )"), + Right( + Expr.and( + Expr.and(Expr.CustomFieldMatch("usd", Operator.Eq, "4.99")), + Expr.or(Expr.string(Operator.Like, Attr.ItemSource, "*test*")) + ) + ) + ) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/ItemQueryParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/ItemQueryParserTest.scala new file mode 100644 index 00000000..52e35fc4 --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/ItemQueryParserTest.scala @@ -0,0 +1,61 @@ +package docspell.query.internal + +import cats.implicits._ + +import munit._ +import docspell.query.ItemQueryParser +import docspell.query.ItemQuery + +class ItemQueryParserTest extends FunSuite { + + test("reduce ands") { + val q = ItemQueryParser.parseUnsafe("(&(&(&(& name:hello))))") + val expr = ExprUtil.reduce(q.expr) + assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr) + } + + test("reduce ors") { + val q = ItemQueryParser.parseUnsafe("(|(|(|(| name:hello))))") + val expr = ExprUtil.reduce(q.expr) + assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr) + } + + test("reduce and/or") { + val q = ItemQueryParser.parseUnsafe("(|(&(&(| name:hello))))") + val expr = ExprUtil.reduce(q.expr) + assertEquals(expr, ItemQueryParser.parseUnsafe("name:hello").expr) + } + + test("reduce inner and/or") { + val q = ItemQueryParser.parseUnsafe("(& name:hello (| name:world))") + val expr = ExprUtil.reduce(q.expr) + assertEquals(expr, ItemQueryParser.parseUnsafe("(& name:hello name:world)").expr) + } + + test("omit and-parens around root structure") { + val q = ItemQueryParser.parseUnsafe("name:hello date>2020-02-02") + val expect = ItemQueryParser.parseUnsafe("(& name:hello date>2020-02-02 )") + assertEquals(expect, q) + } + + test("return all if query is empty") { + val q = ItemQueryParser.parseUnsafe("") + assertEquals(ItemQuery.all, q) + } + + test("splice inner and nodes") { + val raw = "(& name:hello (& date:2021-02 name:world) (& name:hello) )" + val q = ItemQueryParser.parseUnsafe(raw) + val expect = + ItemQueryParser.parseUnsafe("name:hello date:2021-02 name:world name:hello") + assertEquals(expect.copy(raw = raw.some), q) + } + + test("splice inner or nodes") { + val raw = "(| name:hello (| date:2021-02 name:world) (| name:hello) )" + val q = ItemQueryParser.parseUnsafe(raw) + val expect = + ItemQueryParser.parseUnsafe("(| name:hello date:2021-02 name:world name:hello )") + assertEquals(expect.copy(raw = raw.some), q) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/MacroParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/MacroParserTest.scala new file mode 100644 index 00000000..15855916 --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/MacroParserTest.scala @@ -0,0 +1,15 @@ +package docspell.query.internal + +import munit._ +//import cats.parse.{Parser => P} +import docspell.query.ItemQuery.Expr + +class MacroParserTest extends FunSuite { + + test("recognize names shortcut") { + val p = MacroParser.namesMacro + assertEquals(p.parseAll("names:test"), Right(Expr.NamesMacro("test"))) + assert(p.parseAll("$names:test").isLeft) + } + +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/OperatorParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/OperatorParserTest.scala new file mode 100644 index 00000000..1a5a8af0 --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/OperatorParserTest.scala @@ -0,0 +1,24 @@ +package docspell.query.internal + +import munit._ +import docspell.query.ItemQuery.{Operator, TagOperator} +import docspell.query.internal.OperatorParser + +class OperatorParserTest extends FunSuite { + test("operator values") { + val p = OperatorParser.op + assertEquals(p.parseAll("="), Right(Operator.Eq)) + assertEquals(p.parseAll("!="), Right(Operator.Neq)) + assertEquals(p.parseAll(":"), Right(Operator.Like)) + assertEquals(p.parseAll("<"), Right(Operator.Lt)) + assertEquals(p.parseAll(">"), Right(Operator.Gt)) + assertEquals(p.parseAll("<="), Right(Operator.Lte)) + assertEquals(p.parseAll(">="), Right(Operator.Gte)) + } + + test("tag operators") { + val p = OperatorParser.tagOp + assertEquals(p.parseAll(":"), Right(TagOperator.AnyMatch)) + assertEquals(p.parseAll("="), Right(TagOperator.AllMatch)) + } +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala b/modules/query/shared/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala new file mode 100644 index 00000000..35660dfa --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/SimpleExprParserTest.scala @@ -0,0 +1,182 @@ +package docspell.query.internal + +import cats.data.{NonEmptyList => Nel} +import docspell.query.ItemQuery._ +import munit._ +import docspell.query.Date +import java.time.Period + +class SimpleExprParserTest extends FunSuite with ValueHelper { + + test("string expr") { + val p = SimpleExprParser.stringExpr + assertEquals( + p.parseAll("name:hello"), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello")) + ) + assertEquals( + p.parseAll("name: hello"), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello")) + ) + assertEquals( + p.parseAll("name:\"hello world\""), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello world")) + ) + assertEquals( + p.parseAll("name : \"hello world\""), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello world")) + ) + assertEquals( + p.parseAll("conc.pers.id=Aaiet-aied"), + Right(stringExpr(Operator.Eq, Attr.Concerning.PersonId, "Aaiet-aied")) + ) + assert(p.parseAll("conc.pers.id=Aaiet,aied").isLeft) + assertEquals( + p.parseAll("name~=hello,world"), + Right(Expr.InExpr(Attr.ItemName, Nel.of("hello", "world"))) + ) + } + + test("date expr") { + val p = SimpleExprParser.dateExpr + assertEquals( + p.parseAll("date:2021-03-14"), + Right(dateExpr(Operator.Like, Attr.Date, ld(2021, 3, 14))) + ) + assertEquals( + p.parseAll("due<2021-03-14"), + Right(dateExpr(Operator.Lt, Attr.DueDate, ld(2021, 3, 14))) + ) + assertEquals( + p.parseAll("due~=2021-03-14,2021-03-13"), + Right(Expr.InDateExpr(Attr.DueDate, Nel.of(ld(2021, 3, 14), ld(2021, 3, 13)))) + ) + assertEquals( + p.parseAll("due>2021"), + Right(dateExpr(Operator.Gt, Attr.DueDate, ld(2021, 1, 1))) + ) + assertEquals( + p.parseAll("date<2021-01"), + Right(dateExpr(Operator.Lt, Attr.Date, ld(2021, 1, 1))) + ) + assertEquals( + p.parseAll("datetoday;-2m"), + Right( + dateExpr( + Operator.Gt, + Attr.Date, + Date.Calc(Date.Today, Date.CalcDirection.Minus, Period.ofMonths(2)) + ) + ) + ) + } + + test("exists expr") { + val p = SimpleExprParser.existsExpr + assertEquals(p.parseAll("exist:name"), Right(Expr.Exists(Attr.ItemName))) + assert(p.parseAll("exist:blabla").isLeft) + assertEquals( + p.parseAll("exist:conc.pers.id"), + Right(Expr.Exists(Attr.Concerning.PersonId)) + ) + } + + test("fulltext expr") { + val p = SimpleExprParser.fulltextExpr + assertEquals(p.parseAll("content:test"), Right(Expr.Fulltext("test"))) + assertEquals( + p.parseAll("content:\"hello world\""), + Right(Expr.Fulltext("hello world")) + ) + } + + test("category expr") { + val p = SimpleExprParser.catExpr + assertEquals( + p.parseAll("cat:expense,doctype"), + Right(Expr.TagCategoryMatch(TagOperator.AnyMatch, Nel.of("expense", "doctype"))) + ) + } + + test("custom field") { + val p = SimpleExprParser.customFieldExpr + assertEquals( + p.parseAll("f:usd=26.66"), + Right(Expr.CustomFieldMatch("usd", Operator.Eq, "26.66")) + ) + } + + test("tag id expr") { + val p = SimpleExprParser.tagIdExpr + assertEquals( + p.parseAll("tag.id:a,b,c"), + Right(Expr.TagIdsMatch(TagOperator.AnyMatch, Nel.of("a", "b", "c"))) + ) + assertEquals( + p.parseAll("tag.id:a"), + Right(Expr.TagIdsMatch(TagOperator.AnyMatch, Nel.of("a"))) + ) + assertEquals( + p.parseAll("tag.id=a,b,c"), + Right(Expr.TagIdsMatch(TagOperator.AllMatch, Nel.of("a", "b", "c"))) + ) + assertEquals( + p.parseAll("tag.id=a"), + Right(Expr.TagIdsMatch(TagOperator.AllMatch, Nel.of("a"))) + ) + assertEquals( + p.parseAll("tag.id=a,\"x y\""), + Right(Expr.TagIdsMatch(TagOperator.AllMatch, Nel.of("a", "x y"))) + ) + } + + test("simple expr") { + val p = SimpleExprParser.simpleExpr + assertEquals( + p.parseAll("name:hello"), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello")) + ) + assertEquals( + p.parseAll("name:hello"), + Right(stringExpr(Operator.Like, Attr.ItemName, "hello")) + ) + assertEquals( + p.parseAll("due:2021-03-14"), + Right(dateExpr(Operator.Like, Attr.DueDate, ld(2021, 3, 14))) + ) + assertEquals( + p.parseAll("due<2021-03-14"), + Right(dateExpr(Operator.Lt, Attr.DueDate, ld(2021, 3, 14))) + ) + assertEquals( + p.parseAll("exist:conc.pers.id"), + Right(Expr.Exists(Attr.Concerning.PersonId)) + ) + assertEquals(p.parseAll("content:test"), Right(Expr.Fulltext("test"))) + assertEquals( + p.parseAll("tag.id:a"), + Right(Expr.TagIdsMatch(TagOperator.AnyMatch, Nel.of("a"))) + ) + assertEquals( + p.parseAll("tag.id=a,b,c"), + Right(Expr.TagIdsMatch(TagOperator.AllMatch, Nel.of("a", "b", "c"))) + ) + assertEquals( + p.parseAll("cat:expense,doctype"), + Right(Expr.TagCategoryMatch(TagOperator.AnyMatch, Nel.of("expense", "doctype"))) + ) + assertEquals( + p.parseAll("f:usd=26.66"), + Right(Expr.CustomFieldMatch("usd", Operator.Eq, "26.66")) + ) + assertEquals( + p.parseAll("f:usd=\"26.66\""), + Right(Expr.CustomFieldMatch("usd", Operator.Eq, "26.66")) + ) + } + +} diff --git a/modules/query/shared/src/test/scala/docspell/query/internal/ValueHelper.scala b/modules/query/shared/src/test/scala/docspell/query/internal/ValueHelper.scala new file mode 100644 index 00000000..f2729f18 --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/internal/ValueHelper.scala @@ -0,0 +1,24 @@ +package docspell.query.internal + +import docspell.query.Date +import docspell.query.ItemQuery._ +import java.time.Period + +trait ValueHelper { + + def ld(year: Int, m: Int, d: Int): Date.DateLiteral = + Date(year, m, d).fold(throw _, identity) + + def ldPlus(year: Int, m: Int, d: Int, p: Period): Date.Calc = + Date.Calc(ld(year, m, d), Date.CalcDirection.Plus, p) + + def ldMinus(year: Int, m: Int, d: Int, p: Period): Date.Calc = + Date.Calc(ld(year, m, d), Date.CalcDirection.Minus, p) + + def stringExpr(op: Operator, name: Attr.StringAttr, value: String): Expr.SimpleExpr = + Expr.SimpleExpr(op, Property.StringProperty(name, value)) + + def dateExpr(op: Operator, name: Attr.DateAttr, value: Date): Expr.SimpleExpr = + Expr.SimpleExpr(op, Property.DateProperty(name, value)) + +} diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 8623ade1..951f2395 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1310,16 +1310,17 @@ paths: $ref: "#/components/schemas/BasicResult" - /sec/item/search: + /sec/item/searchForm: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items. + deprecated: true description: | Search for items given a search form. The results are grouped by month and are sorted by item date (newest first). Tags and attachments are *not* resolved. The results will always contain an empty list for item tags and attachments. Use - `/searchWithTags` to also retrieve all tags and a list of + `/searchFormWithTags` to also retrieve all tags and a list of attachments of an item. The `fulltext` field can be used to restrict the results by @@ -1328,6 +1329,8 @@ paths: The customfields used in the search query are allowed to be specified by either field id or field name. The values may contain the wildcard `*` at beginning or end. + + **NOTE** This is deprecated in favor for using a search query. security: - authTokenHeader: [] requestBody: @@ -1342,10 +1345,11 @@ paths: application/json: schema: $ref: "#/components/schemas/ItemLightList" - /sec/item/searchWithTags: + /sec/item/searchFormWithTags: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items. + deprecated: true description: | Search for items given a search form. The results are grouped by month by default. For each item, its tags and attachments @@ -1355,6 +1359,8 @@ paths: The `fulltext` field can be used to restrict the results by using full-text search in the documents contents. + + **NOTE** This is deprecated in favor for using search query. security: - authTokenHeader: [] requestBody: @@ -1369,9 +1375,60 @@ paths: application/json: schema: $ref: "#/components/schemas/ItemLightList" + + /sec/item/search: + get: + tags: [ Item Search ] + summary: Search for items. + description: | + Search for items given a search query. The results are grouped + by month and are sorted by item date (newest first). Tags and + attachments are *not* resolved. The results will always + contain an empty list for item tags and attachments. Set + `withDetails` to `true` for retrieving all tags and a list of + attachments of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/q" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/offset" + - $ref: "#/components/parameters/withDetails" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemLightList" + post: + tags: [ Item Search ] + summary: Search for items. + description: | + Search for items given a search query. The results are grouped + by month and are sorted by item date (newest first). Tags and + attachments are *not* resolved. The results will always + contain an empty list for item tags and attachments. Use + `withDetails` to also retrieve all tags and a list of + attachments of an item. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemQuery" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemLightList" + /sec/item/searchIndex: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items using full-text search only. description: | Search for items by only using the full-text search index. @@ -1391,7 +1448,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ItemFtsSearch" + $ref: "#/components/schemas/ItemQuery" responses: 200: description: Ok @@ -1400,12 +1457,15 @@ paths: schema: $ref: "#/components/schemas/ItemLightList" - /sec/item/searchStats: + /sec/item/searchFormStats: post: - tags: [ Item ] + tags: [ Item Search ] summary: Get basic statistics about the data of a search. + deprecated: true description: | Takes a search query and returns a summary about the results. + + **NOTE** This is deprecated in favor of using a search query. security: - authTokenHeader: [] requestBody: @@ -1420,6 +1480,44 @@ paths: application/json: schema: $ref: "#/components/schemas/SearchStats" + /sec/item/searchStats: + post: + tags: [ Item Search ] + summary: Get basic statistics about search results. + description: | + Instead of returning the results of a query, uses it to return + a summary. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemQuery" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" + get: + tags: [ Item Search ] + summary: Get basic statistics about search results. + description: | + Instead of returning the results of a query, uses it to return + a summary. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/q" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" /sec/item/{id}: get: @@ -3777,22 +3875,12 @@ components: type: array items: $ref: "#/components/schemas/IdName" - FolderMember: + ItemQuery: description: | - Information to add or remove a folder member. - required: - - userId - properties: - userId: - type: string - format: ident - ItemFtsSearch: - description: | - Query description for a full-text only search. + Query description for a search. Is used for fulltext-only + searches and combined searches. required: - query - - offset - - limit properties: offset: type: integer @@ -3804,6 +3892,9 @@ components: The maximum number of results to return. Note that this limit is a soft limit, there is some hard limit on the server, too. + withDetails: + type: boolean + default: false query: type: string description: | @@ -5547,6 +5638,26 @@ components: required: false schema: type: string + limit: + name: limit + in: query + description: A limit for a search query + schema: + type: integer + format: int32 + offset: + name: offset + in: query + description: A offset into the results for a search query + schema: + type: integer + format: int32 + withDetails: + name: withDetails + in: query + description: Whether to return details to each item. + schema: + type: boolean name: name: name in: path diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 4aebc8c8..cede3845 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -145,31 +145,32 @@ trait Conversions { def mkQuery(m: ItemSearch, account: AccountId): OItemSearch.Query = OItemSearch.Query( - account, - m.name, - if (m.inbox) Seq(ItemState.Created) - else ItemState.validStates.toList, - m.direction, - m.corrPerson, - m.corrOrg, - m.concPerson, - m.concEquip, - m.folder, - m.tagsInclude.map(Ident.unsafe), - m.tagsExclude.map(Ident.unsafe), - m.tagCategoriesInclude, - m.tagCategoriesExclude, - m.dateFrom, - m.dateUntil, - m.dueDateFrom, - m.dueDateUntil, - m.allNames, - m.itemSubset - .map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet) - .filter(_.nonEmpty), - m.customValues.map(mkCustomValue), - m.source, - None + OItemSearch.Query.Fix(account, None, None), + OItemSearch.Query.QueryForm( + m.name, + if (m.inbox) Seq(ItemState.Created) + else ItemState.validStates.toList, + m.direction, + m.corrPerson, + m.corrOrg, + m.concPerson, + m.concEquip, + m.folder, + m.tagsInclude.map(Ident.unsafe), + m.tagsExclude.map(Ident.unsafe), + m.tagCategoriesInclude, + m.tagCategoriesExclude, + m.dateFrom, + m.dateUntil, + m.dueDateFrom, + m.dueDateUntil, + m.allNames, + m.itemSubset + .map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet) + .filter(_.nonEmpty), + m.customValues.map(mkCustomValue), + m.source + ) ) def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue = @@ -182,7 +183,7 @@ trait Conversions { ItemLightGroup(g._1, g._2.map(mkItemLight).toList) val gs = - groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) + groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) ItemLightList(gs) } @@ -223,6 +224,10 @@ trait Conversions { if (v.isEmpty) ItemLightList(Nil) else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList))) + def mkItemListFtsPlain(v: Vector[OFulltext.FtsItem]): ItemLightList = + if (v.isEmpty) ItemLightList(Nil) + else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLight).toList))) + def mkItemLight(i: OItemSearch.ListItem): ItemLight = ItemLight( i.id, diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala index aa846c7e..6907c2c4 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala @@ -25,5 +25,10 @@ object QueryParam { object QueryOpt extends OptionalQueryParamDecoderMatcher[QueryString]("q") + object Query extends OptionalQueryParamDecoderMatcher[String]("q") + object Limit extends OptionalQueryParamDecoderMatcher[Int]("limit") + object Offset extends OptionalQueryParamDecoderMatcher[Int]("offset") + object WithDetails extends OptionalQueryParamDecoderMatcher[Boolean]("withDetails") + object WithFallback extends OptionalQueryParamDecoderMatcher[Boolean]("withFallback") } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index e0b0c5b8..2e7fbbb5 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -9,9 +9,13 @@ import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OFulltext -import docspell.backend.ops.OItemSearch.Batch +import docspell.backend.ops.OItemSearch.{Batch, Query} +import docspell.backend.ops.OSimpleSearch +import docspell.backend.ops.OSimpleSearch.StringSearchResult import docspell.common._ import docspell.common.syntax.all._ +import docspell.query.FulltextExtract.Result.TooMany +import docspell.query.FulltextExtract.Result.UnsupportedPosition import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -46,7 +50,60 @@ object ItemRoutes { resp <- Ok(Conversions.basicResult(res, "Task submitted")) } yield resp + case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( + offset + ) :? QP.WithDetails(detailFlag) => + val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize)) + .restrictLimitTo(cfg.maxItemPageSize) + val itemQuery = ItemQueryString(q) + val settings = OSimpleSearch.Settings( + batch, + cfg.fullTextSearch.enabled, + detailFlag.getOrElse(false), + cfg.maxNoteLength + ) + val fixQuery = Query.Fix(user.account, None, None) + searchItems(backend, dsl)(settings, fixQuery, itemQuery) + + case GET -> Root / "searchStats" :? QP.Query(q) => + val itemQuery = ItemQueryString(q) + val fixQuery = Query.Fix(user.account, None, None) + searchItemStats(backend, dsl)(cfg.fullTextSearch.enabled, fixQuery, itemQuery) + case req @ POST -> Root / "search" => + for { + userQuery <- req.as[ItemQuery] + batch = Batch( + userQuery.offset.getOrElse(0), + userQuery.limit.getOrElse(cfg.maxItemPageSize) + ).restrictLimitTo( + cfg.maxItemPageSize + ) + itemQuery = ItemQueryString(userQuery.query) + settings = OSimpleSearch.Settings( + batch, + cfg.fullTextSearch.enabled, + userQuery.withDetails.getOrElse(false), + cfg.maxNoteLength + ) + fixQuery = Query.Fix(user.account, None, None) + resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery) + } yield resp + + case req @ POST -> Root / "searchStats" => + for { + userQuery <- req.as[ItemQuery] + itemQuery = ItemQueryString(userQuery.query) + fixQuery = Query.Fix(user.account, None, None) + resp <- searchItemStats(backend, dsl)( + cfg.fullTextSearch.enabled, + fixQuery, + itemQuery + ) + } yield resp + + //DEPRECATED + case req @ POST -> Root / "searchForm" => for { mask <- req.as[ItemSearch] _ <- logger.ftrace(s"Got search mask: $mask") @@ -85,7 +142,8 @@ object ItemRoutes { } } yield resp - case req @ POST -> Root / "searchWithTags" => + //DEPRECATED + case req @ POST -> Root / "searchFormWithTags" => for { mask <- req.as[ItemSearch] _ <- logger.ftrace(s"Got search mask: $mask") @@ -125,7 +183,7 @@ object ItemRoutes { case req @ POST -> Root / "searchIndex" => for { - mask <- req.as[ItemFtsSearch] + mask <- req.as[ItemQuery] resp <- mask.query match { case q if q.length > 1 => val ftsIn = OFulltext.FtsInput(q) @@ -133,7 +191,10 @@ object ItemRoutes { items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( ftsIn, user.account, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) + Batch( + mask.offset.getOrElse(0), + mask.limit.getOrElse(cfg.maxItemPageSize) + ).restrictLimitTo(cfg.maxItemPageSize) ) ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) } yield ok @@ -143,7 +204,8 @@ object ItemRoutes { } } yield resp - case req @ POST -> Root / "searchStats" => + //DEPRECATED + case req @ POST -> Root / "searchFormStats" => for { mask <- req.as[ItemSearch] query = Conversions.mkQuery(mask, user.account) @@ -429,6 +491,71 @@ object ItemRoutes { } } + def searchItems[F[_]: Sync]( + backend: BackendApp[F], + dsl: Http4sDsl[F] + )(settings: OSimpleSearch.Settings, fixQuery: Query.Fix, itemQuery: ItemQueryString) = { + import dsl._ + + def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList = + if (res.indexOnly) Conversions.mkItemListFtsPlain(res.items) + else Conversions.mkItemListFts(res.items) + + def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList = + if (res.indexOnly) Conversions.mkItemListWithTagsFtsPlain(res.items) + else Conversions.mkItemListWithTagsFts(res.items) + + backend.simpleSearch + .searchByString(settings)(fixQuery, itemQuery) + .flatMap { + case StringSearchResult.Success(items) => + Ok( + items.fold( + convertFts, + convertFtsFull, + Conversions.mkItemList, + Conversions.mkItemListWithTags + ) + ) + case StringSearchResult.FulltextMismatch(TooMany) => + BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) + case StringSearchResult.FulltextMismatch(UnsupportedPosition) => + BadRequest( + BasicResult( + false, + "Fulltext search must be in root position or inside the first AND." + ) + ) + case StringSearchResult.ParseFailed(pf) => + BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) + } + } + + def searchItemStats[F[_]: Sync]( + backend: BackendApp[F], + dsl: Http4sDsl[F] + )(ftsEnabled: Boolean, fixQuery: Query.Fix, itemQuery: ItemQueryString) = { + import dsl._ + backend.simpleSearch + .searchSummaryByString(ftsEnabled)(fixQuery, itemQuery) + .flatMap { + case StringSearchResult.Success(summary) => + Ok(Conversions.mkSearchStats(summary)) + case StringSearchResult.FulltextMismatch(TooMany) => + BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) + case StringSearchResult.FulltextMismatch(UnsupportedPosition) => + BadRequest( + BasicResult( + false, + "Fulltext search must be in root position or inside the first AND." + ) + ) + case StringSearchResult.ParseFailed(pf) => + BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) + } + + } + implicit final class OptionString(opt: Option[String]) { def notEmpty: Option[String] = opt.map(_.trim).filter(_.nonEmpty) @@ -453,12 +580,12 @@ object ItemRoutes { private val itemSearchMonoid: Monoid[ItemSearch] = cats.derived.semiauto.monoid - def unapply(m: ItemSearch): Option[ItemFtsSearch] = + def unapply(m: ItemSearch): Option[ItemQuery] = m.fullText match { case Some(fq) => val me = m.copy(fullText = None, offset = 0, limit = 0) if (itemSearchMonoid.empty == me) - Some(ItemFtsSearch(m.offset, m.limit, fq)) + Some(ItemQuery(m.offset.some, m.limit.some, Some(false), fq)) else None case _ => None diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala index b920168a..9d59c201 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala @@ -170,7 +170,8 @@ object TemplateRoutes { chooseUi(uiVersion), Seq( "/app/assets" + Webjars.clipboardjs + "/clipboard.min.js", - s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js" + s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js", + s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-query-opt.js" ), s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon", s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js", diff --git a/modules/store/src/main/resources/db/migration/h2/V1.21.0__cast_function.sql b/modules/store/src/main/resources/db/migration/h2/V1.21.0__cast_function.sql new file mode 100644 index 00000000..146bf3d8 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.21.0__cast_function.sql @@ -0,0 +1,12 @@ +DROP ALIAS IF EXISTS CAST_TO_NUMERIC; +CREATE ALIAS CAST_TO_NUMERIC AS ' +import java.text.*; +import java.math.*; +@CODE +BigDecimal castToNumeric(String s) throws Exception { + try { return new BigDecimal(s); } + catch (Exception e) { + return null; + } +} +' diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.21.0__cast_function.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.21.0__cast_function.sql new file mode 100644 index 00000000..24ae76de --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.21.0__cast_function.sql @@ -0,0 +1,5 @@ +-- Create a function to cast to a numeric, if an error occurs return null +-- Could not get it working with decimal type, so using double +create or replace function CAST_TO_NUMERIC (s char(255)) +returns double deterministic +return cast(s as double); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.21.0__cast_function.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.21.0__cast_function.sql new file mode 100644 index 00000000..b603275c --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.21.0__cast_function.sql @@ -0,0 +1,9 @@ +-- Create a function to cast to a numeric, if an error occurs return null +create or replace function CAST_TO_NUMERIC(text) returns numeric as $$ +begin + return cast($1 as numeric); +exception + when invalid_text_representation then + return null; +end; +$$ language plpgsql immutable; diff --git a/modules/store/src/main/scala/docspell/store/qb/Column.scala b/modules/store/src/main/scala/docspell/store/qb/Column.scala index 3e59a62c..e5e13749 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Column.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Column.scala @@ -3,6 +3,9 @@ package docspell.store.qb case class Column[A](name: String, table: TableDef) { def inTable(t: TableDef): Column[A] = copy(table = t) + + def cast[B]: Column[B] = + this.asInstanceOf[Column[B]] } object Column {} diff --git a/modules/store/src/main/scala/docspell/store/qb/Condition.scala b/modules/store/src/main/scala/docspell/store/qb/Condition.scala index 0b0c0692..9a329033 100644 --- a/modules/store/src/main/scala/docspell/store/qb/Condition.scala +++ b/modules/store/src/main/scala/docspell/store/qb/Condition.scala @@ -15,7 +15,7 @@ object Condition { val P: Put[A] ) extends Condition - case class CompareFVal[A](dbf: DBFunction, op: Operator, value: A)(implicit + case class CompareFVal[A](sel: SelectExpr, op: Operator, value: A)(implicit val P: Put[A] ) extends Condition @@ -23,11 +23,11 @@ object Condition { extends Condition case class InSubSelect[A](col: Column[A], subSelect: Select) extends Condition - case class InValues[A](col: Column[A], values: NonEmptyList[A], lower: Boolean)(implicit - val P: Put[A] + case class InValues[A](sel: SelectExpr, values: NonEmptyList[A], lower: Boolean)( + implicit val P: Put[A] ) extends Condition - case class IsNull(col: Column[_]) extends Condition + case class IsNull(sel: SelectExpr) extends Condition case class And(inner: NonEmptyList[Condition]) extends Condition { def append(other: Condition): And = diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 58cca850..40db91b2 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -29,6 +29,8 @@ object DBFunction { case class Cast(expr: SelectExpr, newType: String) extends DBFunction + case class CastNumeric(expr: SelectExpr) extends DBFunction + case class Avg(expr: SelectExpr) extends DBFunction case class Sum(expr: SelectExpr) extends DBFunction diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index e90439bf..d2e78be1 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -89,6 +89,9 @@ trait DSL extends DoobieMeta { def cast(expr: SelectExpr, targetType: String): DBFunction = DBFunction.Cast(expr, targetType) + def castNumeric(expr: SelectExpr): DBFunction = + DBFunction.CastNumeric(expr) + def coalesce(expr: SelectExpr, more: SelectExpr*): DBFunction.Coalesce = DBFunction.Coalesce(expr, more.toVector) @@ -174,13 +177,13 @@ trait DSL extends DoobieMeta { Condition.CompareVal(col, Operator.LowerEq, value) def ====(value: String): Condition = - Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.Eq, value) + Condition.CompareVal(col.cast[String], Operator.Eq, value) def like(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.LowerLike, value) def likes(value: String): Condition = - Condition.CompareVal(col.asInstanceOf[Column[String]], Operator.LowerLike, value) + Condition.CompareVal(col.cast[String], Operator.LowerLike, value) def <=(value: A)(implicit P: Put[A]): Condition = Condition.CompareVal(col, Operator.Lte, value) @@ -204,22 +207,22 @@ trait DSL extends DoobieMeta { in(subsel).negate def in(values: Nel[A])(implicit P: Put[A]): Condition = - Condition.InValues(col, values, false) + Condition.InValues(col.s, values, false) def notIn(values: Nel[A])(implicit P: Put[A]): Condition = in(values).negate def inLower(values: Nel[A])(implicit P: Put[A]): Condition = - Condition.InValues(col, values, true) + Condition.InValues(col.s, values, true) def notInLower(values: Nel[A])(implicit P: Put[A]): Condition = - Condition.InValues(col, values, true).negate + Condition.InValues(col.s, values, true).negate def isNull: Condition = - Condition.IsNull(col) + Condition.IsNull(col.s) def isNotNull: Condition = - Condition.IsNull(col).negate + Condition.IsNull(col.s).negate def ===(other: Column[A]): Condition = Condition.CompareCol(col, Operator.Eq, other) @@ -264,31 +267,31 @@ trait DSL extends DoobieMeta { SelectExpr.SelectFun(dbf, Some(otherCol.name)) def ===[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Eq, value) + Condition.CompareFVal(dbf.s, Operator.Eq, value) def ====(value: String): Condition = - Condition.CompareFVal(dbf, Operator.Eq, value) + Condition.CompareFVal(dbf.s, Operator.Eq, value) def like[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.LowerLike, value) + Condition.CompareFVal(dbf.s, Operator.LowerLike, value) def likes(value: String): Condition = - Condition.CompareFVal(dbf, Operator.LowerLike, value) + Condition.CompareFVal(dbf.s, Operator.LowerLike, value) def <=[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Lte, value) + Condition.CompareFVal(dbf.s, Operator.Lte, value) def >=[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Gte, value) + Condition.CompareFVal(dbf.s, Operator.Gte, value) def >[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Gt, value) + Condition.CompareFVal(dbf.s, Operator.Gt, value) def <[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Lt, value) + Condition.CompareFVal(dbf.s, Operator.Lt, value) def <>[A](value: A)(implicit P: Put[A]): Condition = - Condition.CompareFVal(dbf, Operator.Neq, value) + Condition.CompareFVal(dbf.s, Operator.Neq, value) def -[A](value: A)(implicit P: Put[A]): DBFunction = DBFunction.Calc( @@ -297,6 +300,35 @@ trait DSL extends DoobieMeta { SelectExpr.SelectConstant(value, None) ) } + + implicit final class SelectExprOps(sel: SelectExpr) { + def isNull: Condition = + Condition.IsNull(sel) + + def isNotNull: Condition = + Condition.IsNull(sel).negate + + def ===[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Eq, value) + + def <=[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Lte, value) + + def >=[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Gte, value) + + def >[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Gt, value) + + def <[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Lt, value) + + def <>[A](value: A)(implicit P: Put[A]): Condition = + Condition.CompareFVal(sel, Operator.Neq, value) + + def in[A](values: Nel[A])(implicit P: Put[A]): Condition = + Condition.InValues(sel, values, false) + } } object DSL extends DSL { diff --git a/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala new file mode 100644 index 00000000..6a721270 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala @@ -0,0 +1,299 @@ +package docspell.store.qb.generator + +import java.time.Instant +import java.time.LocalDate + +import cats.data.{NonEmptyList => Nel} + +import docspell.common._ +import docspell.query.ItemQuery._ +import docspell.query.{Date, ItemQuery} +import docspell.store.qb.DSL._ +import docspell.store.qb.{Operator => QOp, _} +import docspell.store.queries.QItem +import docspell.store.queries.QueryWildcard +import docspell.store.records._ + +import doobie.util.Put + +object ItemQueryGenerator { + + def apply(today: LocalDate, tables: Tables, coll: Ident)(q: ItemQuery)(implicit + PT: Put[Timestamp] + ): Condition = + fromExpr(today, tables, coll)(q.expr) + + final def fromExpr(today: LocalDate, tables: Tables, coll: Ident)( + expr: Expr + )(implicit PT: Put[Timestamp]): Condition = + expr match { + case Expr.AndExpr(inner) => + Condition.And(inner.map(fromExpr(today, tables, coll))) + + case Expr.OrExpr(inner) => + Condition.Or(inner.map(fromExpr(today, tables, coll))) + + case Expr.NotExpr(inner) => + inner match { + case Expr.Exists(notExists) => + anyColumn(tables)(notExists).isNull + + case Expr.TagIdsMatch(op, tags) => + val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption) + Nel + .fromList(ids) + .map { nel => + op match { + case TagOperator.AnyMatch => + tables.item.id.notIn(TagItemName.itemsWithEitherTag(nel)) + case TagOperator.AllMatch => + tables.item.id.notIn(TagItemName.itemsWithAllTags(nel)) + } + } + .getOrElse(Condition.unit) + case Expr.TagsMatch(op, tags) => + op match { + case TagOperator.AllMatch => + tables.item.id.notIn(TagItemName.itemsWithAllTagNameOrIds(tags)) + + case TagOperator.AnyMatch => + tables.item.id.notIn(TagItemName.itemsWithEitherTagNameOrIds(tags)) + } + + case Expr.TagCategoryMatch(op, cats) => + op match { + case TagOperator.AllMatch => + tables.item.id.notIn(TagItemName.itemsInAllCategories(cats)) + + case TagOperator.AnyMatch => + tables.item.id.notIn(TagItemName.itemsInEitherCategory(cats)) + } + + case Expr.Fulltext(_) => + Condition.unit + + case _ => + Condition.Not(fromExpr(today, tables, coll)(inner)) + } + + case Expr.Exists(field) => + anyColumn(tables)(field).isNotNull + + case Expr.SimpleExpr(op, Property.StringProperty(attr, value)) => + val col = stringColumn(tables)(attr) + op match { + case Operator.Like => + Condition.CompareVal(col, makeOp(op), QueryWildcard.lower(value)) + case _ => + Condition.CompareVal(col, makeOp(op), value) + } + + case Expr.SimpleExpr(op, Property.DateProperty(attr, value)) => + val dt = dateToTimestamp(today)(value) + val col = timestampColumn(tables)(attr) + val noLikeOp = if (op == Operator.Like) Operator.Eq else op + Condition.CompareFVal(col, makeOp(noLikeOp), dt) + + case Expr.SimpleExpr(op, Property.IntProperty(attr, value)) => + val col = intColumn(tables)(attr) + Condition.CompareVal(col, makeOp(op), value) + + case Expr.InExpr(attr, values) => + val col = stringColumn(tables)(attr) + if (values.tail.isEmpty) col === values.head + else col.in(values) + + case Expr.InDateExpr(attr, values) => + val col = timestampColumn(tables)(attr) + val dts = values.map(dateToTimestamp(today)) + if (values.tail.isEmpty) col === dts.head + else col.in(dts) + + case Expr.DirectionExpr(incoming) => + if (incoming) tables.item.incoming === Direction.Incoming + else tables.item.incoming === Direction.Outgoing + + case Expr.InboxExpr(flag) => + if (flag) tables.item.state === ItemState.created + else tables.item.state === ItemState.confirmed + + case Expr.TagIdsMatch(op, tags) => + val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption) + Nel + .fromList(ids) + .map { nel => + op match { + case TagOperator.AnyMatch => + tables.item.id.in(TagItemName.itemsWithEitherTag(nel)) + case TagOperator.AllMatch => + tables.item.id.in(TagItemName.itemsWithAllTags(nel)) + } + } + .getOrElse(Condition.unit) + + case Expr.TagsMatch(op, tags) => + op match { + case TagOperator.AllMatch => + tables.item.id.in(TagItemName.itemsWithAllTagNameOrIds(tags)) + + case TagOperator.AnyMatch => + tables.item.id.in(TagItemName.itemsWithEitherTagNameOrIds(tags)) + } + + case Expr.TagCategoryMatch(op, cats) => + op match { + case TagOperator.AllMatch => + tables.item.id.in(TagItemName.itemsInAllCategories(cats)) + + case TagOperator.AnyMatch => + tables.item.id.in(TagItemName.itemsInEitherCategory(cats)) + } + + case Expr.CustomFieldMatch(field, op, value) => + tables.item.id.in( + itemsWithCustomField(_.name ==== field)(coll, makeOp(op), value) + ) + + case Expr.CustomFieldIdMatch(field, op, value) => + tables.item.id.in(itemsWithCustomField(_.id ==== field)(coll, makeOp(op), value)) + + case Expr.ChecksumMatch(checksum) => + val select = QItem.findByChecksumQuery(checksum, coll, Set.empty) + tables.item.id.in(select.withSelect(Nel.of(RItem.as("i").id.s))) + + case Expr.AttachId(id) => + tables.item.id.in( + Select( + select(RAttachment.T.itemId), + from(RAttachment.T), + RAttachment.T.id.cast[String] === id + ).distinct + ) + + case Expr.Fulltext(_) => + // not supported here + Condition.unit + + case _: Expr.MacroExpr => + Condition.unit + } + + private def dateToTimestamp(today: LocalDate)(date: Date): Timestamp = + date match { + case d: Date.DateLiteral => + val ld = dateLiteralToDate(today)(d) + Timestamp.atUtc(ld.atStartOfDay) + case Date.Calc(date, c, period) => + val ld = c match { + case Date.CalcDirection.Plus => + dateLiteralToDate(today)(date).plus(period) + case Date.CalcDirection.Minus => + dateLiteralToDate(today)(date).minus(period) + } + Timestamp.atUtc(ld.atStartOfDay()) + } + + private def dateLiteralToDate(today: LocalDate)(dateLit: Date.DateLiteral): LocalDate = + dateLit match { + case Date.Local(date) => + date + case Date.Millis(ms) => + Instant.ofEpochMilli(ms).atZone(Timestamp.UTC).toLocalDate() + case Date.Today => + today + } + + private def anyColumn(tables: Tables)(attr: Attr): SelectExpr = + attr match { + case s: Attr.StringAttr => + stringColumn(tables)(s).s + case t: Attr.DateAttr => + timestampColumn(tables)(t) + case n: Attr.IntAttr => + intColumn(tables)(n).s + } + + private def timestampColumn(tables: Tables)(attr: Attr.DateAttr): SelectExpr = + attr match { + case Attr.Date => + coalesce(tables.item.itemDate.s, tables.item.created.s).s + case Attr.DueDate => + tables.item.dueDate.s + } + + private def stringColumn(tables: Tables)(attr: Attr.StringAttr): Column[String] = + attr match { + case Attr.ItemId => tables.item.id.cast[String] + case Attr.ItemName => tables.item.name + case Attr.ItemSource => tables.item.source + case Attr.ItemNotes => tables.item.notes + case Attr.Correspondent.OrgId => tables.corrOrg.oid.cast[String] + case Attr.Correspondent.OrgName => tables.corrOrg.name + case Attr.Correspondent.PersonId => tables.corrPers.pid.cast[String] + case Attr.Correspondent.PersonName => tables.corrPers.name + case Attr.Concerning.PersonId => tables.concPers.pid.cast[String] + case Attr.Concerning.PersonName => tables.concPers.name + case Attr.Concerning.EquipId => tables.concEquip.eid.cast[String] + case Attr.Concerning.EquipName => tables.concEquip.name + case Attr.Folder.FolderId => tables.folder.id.cast[String] + case Attr.Folder.FolderName => tables.folder.name + } + + private def intColumn(tables: Tables)(attr: Attr.IntAttr): Column[Int] = + attr match { + case Attr.AttachCount => tables.attachCount.num + } + + private def makeOp(operator: Operator): QOp = + operator match { + case Operator.Eq => + QOp.Eq + case Operator.Neq => + QOp.Neq + case Operator.Like => + QOp.LowerLike + case Operator.Gt => + QOp.Gt + case Operator.Lt => + QOp.Lt + case Operator.Gte => + QOp.Gte + case Operator.Lte => + QOp.Lte + } + + private def itemsWithCustomField( + sel: RCustomField.Table => Condition + )(coll: Ident, op: QOp, value: String): Select = { + val cf = RCustomField.as("cf") + val cfv = RCustomFieldValue.as("cfv") + + val baseSelect = + Select( + select(cfv.itemId), + from(cfv).innerJoin(cf, sel(cf) && cf.cid === coll && cf.id === cfv.field) + ) + + if (op == QOp.LowerLike) { + val v = QueryWildcard.lower(value) + baseSelect.where(Condition.CompareVal(cfv.value, op, v)) + } else { + val stringCmp = + Condition.CompareVal(cfv.value, op, value) + + value.toDoubleOption + .map { n => + val numericCmp = Condition.CompareFVal(castNumeric(cfv.value.s).s, op, n) + val fieldIsNumeric = + cf.ftype === CustomFieldType.Numeric || cf.ftype === CustomFieldType.Money + val fieldNotNumeric = + cf.ftype <> CustomFieldType.Numeric && cf.ftype <> CustomFieldType.Money + baseSelect.where( + (fieldIsNumeric && numericCmp) || (fieldNotNumeric && stringCmp) + ) + } + .getOrElse(baseSelect.where(stringCmp)) + } + } + +} diff --git a/modules/store/src/main/scala/docspell/store/qb/generator/Tables.scala b/modules/store/src/main/scala/docspell/store/qb/generator/Tables.scala new file mode 100644 index 00000000..0d30c99c --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/qb/generator/Tables.scala @@ -0,0 +1,16 @@ +package docspell.store.qb.generator + +import docspell.store.queries.AttachCountTable +import docspell.store.records._ + +final case class Tables( + item: RItem.Table, + corrOrg: ROrganization.Table, + corrPers: RPerson.Table, + concPers: RPerson.Table, + concEquip: REquipment.Table, + folder: RFolder.Table, + attach: RAttachment.Table, + meta: RAttachmentMeta.Table, + attachCount: AttachCountTable +) diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala index f110bb92..c9e50575 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/ConditionBuilder.scala @@ -85,7 +85,7 @@ object ConditionBuilder { case Operator.LowerEq => lower(dbf) case _ => - DBFunctionBuilder.build(dbf) + SelectExprBuilder.build(dbf) } dbfFrag ++ opFrag ++ valFrag @@ -105,13 +105,13 @@ object ConditionBuilder { SelectExprBuilder.column(col) ++ sql" IN (" ++ sub ++ parenClose case c @ Condition.InValues(col, values, toLower) => - val cfrag = if (toLower) lower(col) else SelectExprBuilder.column(col) + val cfrag = if (toLower) lower(col) else SelectExprBuilder.build(col) cfrag ++ sql" IN (" ++ values.toList .map(a => buildValue(a)(c.P)) .reduce(_ ++ comma ++ _) ++ parenClose case Condition.IsNull(col) => - SelectExprBuilder.column(col) ++ fr" is null" + SelectExprBuilder.build(col) ++ fr" is null" case Condition.And(ands) => val inner = ands.map(build).reduceLeft(_ ++ and ++ _) @@ -124,7 +124,7 @@ object ConditionBuilder { else parenOpen ++ inner ++ parenClose case Condition.Not(Condition.IsNull(col)) => - SelectExprBuilder.column(col) ++ fr" is not null" + SelectExprBuilder.build(col) ++ fr" is not null" case Condition.Not(c) => fr"NOT" ++ build(c) @@ -159,6 +159,9 @@ object ConditionBuilder { def buildOptValue[A: Put](v: Option[A]): Fragment = fr"$v" + def lower(sel: SelectExpr): Fragment = + Fragment.const0("LOWER(") ++ SelectExprBuilder.build(sel) ++ parenClose + def lower(col: Column[_]): Fragment = Fragment.const0("LOWER(") ++ SelectExprBuilder.column(col) ++ parenClose diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index 3a75569a..c57a9ac3 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -46,6 +46,9 @@ object DBFunctionBuilder extends CommonBuilder { fr" AS" ++ Fragment.const(newType) ++ sql")" + case DBFunction.CastNumeric(f) => + sql"CAST_TO_NUMERIC(" ++ SelectExprBuilder.build(f) ++ sql")" + case DBFunction.Avg(expr) => sql"AVG(" ++ SelectExprBuilder.build(expr) ++ fr")" diff --git a/modules/store/src/main/scala/docspell/store/queries/AttachCountTable.scala b/modules/store/src/main/scala/docspell/store/queries/AttachCountTable.scala new file mode 100644 index 00000000..2ed6fc3c --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/AttachCountTable.scala @@ -0,0 +1,16 @@ +package docspell.store.queries + +import docspell.common.Ident +import docspell.store.qb.Column +import docspell.store.qb.TableDef + +final case class AttachCountTable(aliasName: String) extends TableDef { + val tableName = "attachs" + val alias: Option[String] = Some(aliasName) + + val num = Column[Int]("num", this) + val itemId = Column[Ident]("item_id", this) + + def as(alias: String): AttachCountTable = + copy(aliasName = alias) +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 751d5706..c1ee5f2c 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -1,5 +1,7 @@ package docspell.store.queries +import java.time.LocalDate + import cats.data.{NonEmptyList => Nel} import cats.effect.Sync import cats.effect.concurrent.Ref @@ -8,9 +10,11 @@ import fs2.Stream import docspell.common.syntax.all._ import docspell.common.{IdRef, _} +import docspell.query.ItemQuery import docspell.store.Store import docspell.store.qb.DSL._ import docspell.store.qb._ +import docspell.store.qb.generator.{ItemQueryGenerator, Tables} import docspell.store.records._ import doobie.implicits._ @@ -117,18 +121,11 @@ object QItem { .map(nel => intersect(nel.map(singleSelect))) } - private def findItemsBase(q: Query, noteMaxLen: Int): Select = { - object Attachs extends TableDef { - val tableName = "attachs" - val aliasName = "cta" - val alias = Some(aliasName) - val num = Column[Int]("num", this) - val itemId = Column[Ident]("item_id", this) - } + private def findItemsBase(q: Query.Fix, noteMaxLen: Int): Select = { + val attachs = AttachCountTable("cta") + val coll = q.account.collective - val coll = q.account.collective - - val baseSelect = Select( + Select( select( i.id.s, i.name.s, @@ -138,7 +135,7 @@ object QItem { i.source.s, i.incoming.s, i.created.s, - coalesce(Attachs.num.s, const(0)).s, + coalesce(attachs.num.s, const(0)).s, org.oid.s, org.name.s, pers0.pid.s, @@ -158,41 +155,40 @@ object QItem { .leftJoin(f, f.id === i.folder && f.collective === coll) .leftJoin( Select( - select(countAll.as(Attachs.num), a.itemId.as(Attachs.itemId)), + select(countAll.as(attachs.num), a.itemId.as(attachs.itemId)), from(a) .innerJoin(i, i.id === a.itemId), i.cid === q.account.collective, GroupBy(a.itemId) ), - Attachs.aliasName, - Attachs.itemId === i.id + attachs.aliasName, + attachs.itemId === i.id ) .leftJoin(pers0, pers0.pid === i.corrPerson && pers0.cid === coll) .leftJoin(org, org.oid === i.corrOrg && org.cid === coll) .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll) .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll), where( - i.cid === coll &&? Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) && - or(i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account))) + i.cid === coll &&? q.itemIds.map(s => + Nel.fromList(s.toList).map(nel => i.id.in(nel)).getOrElse(i.id.isNull) + ) + && or( + i.folder.isNull, + i.folder.in(QFolder.findMemberFolderIds(q.account)) + ) ) ).distinct.orderBy( q.orderAsc .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s)) .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s)) ) - - findCustomFieldValuesForColl(coll, q.customValues) match { - case Some(itemIds) => - baseSelect.changeWhere(c => c && i.id.in(itemIds)) - case None => - baseSelect - } } - def queryCondition(q: Query): Condition = + def queryCondFromForm(coll: Ident, q: Query.QueryForm): Condition = Condition.unit &&? q.direction.map(d => i.incoming === d) &&? q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? + Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&? q.allNames .map(QueryWildcard.lower) .map(n => @@ -221,40 +217,56 @@ object QItem { .map(subsel => i.id.in(subsel)) &&? TagItemName .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) - .map(subsel => i.id.notIn(subsel)) + .map(subsel => i.id.notIn(subsel)) &&? + findCustomFieldValuesForColl(coll, q.customValues) + .map(itemIds => i.id.in(itemIds)) + + def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery): Condition = { + val tables = Tables(i, org, pers0, pers1, equip, f, a, m, AttachCountTable("cta")) + ItemQueryGenerator.fromExpr(today, tables, coll)(q.expr) + } + + def queryCondition(today: LocalDate, coll: Ident, cond: Query.QueryCond): Condition = + cond match { + case fm: Query.QueryForm => + queryCondFromForm(coll, fm) + case expr: Query.QueryExpr => + queryCondFromExpr(today, coll, expr.q) + } def findItems( q: Query, + today: LocalDate, maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - val sql = findItemsBase(q, maxNoteLen) - .changeWhere(c => c && queryCondition(q)) + val sql = findItemsBase(q.fix, maxNoteLen) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .limit(batch) .build logger.trace(s"List $batch items: $sql") sql.query[ListItem].stream } - def searchStats(q: Query): ConnectionIO[SearchSummary] = + def searchStats(today: LocalDate)(q: Query): ConnectionIO[SearchSummary] = for { - count <- searchCountSummary(q) - tags <- searchTagSummary(q) - fields <- searchFieldSummary(q) - folders <- searchFolderSummary(q) + count <- searchCountSummary(today)(q) + tags <- searchTagSummary(today)(q) + fields <- searchFieldSummary(today)(q) + folders <- searchFolderSummary(today)(q) } yield SearchSummary(count, tags, fields, folders) - def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { + def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = from(ti) .innerJoin(tag, tag.tid === ti.tagId) .innerJoin(i, i.id === ti.itemId) val tagCloud = - findItemsBase(q, 0).unwrap + findItemsBase(q.fix, 0).unwrap .withSelect(select(tag.all).append(count(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) - .changeWhere(c => c && queryCondition(q)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(tag.tid) .build .query[TagCount] @@ -264,40 +276,40 @@ object QItem { // are not included they are fetched separately for { existing <- tagCloud - other <- RTag.findOthers(q.account.collective, existing.map(_.tag.tagId)) + other <- RTag.findOthers(q.fix.account.collective, existing.map(_.tag.tagId)) } yield existing ++ other.map(TagCount(_, 0)) } - def searchCountSummary(q: Query): ConnectionIO[Int] = - findItemsBase(q, 0).unwrap + def searchCountSummary(today: LocalDate)(q: Query): ConnectionIO[Int] = + findItemsBase(q.fix, 0).unwrap .withSelect(Nel.of(count(i.id).as("num"))) - .changeWhere(c => c && queryCondition(q)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .build .query[Int] .unique - def searchFolderSummary(q: Query): ConnectionIO[List[FolderCount]] = { + def searchFolderSummary(today: LocalDate)(q: Query): ConnectionIO[List[FolderCount]] = { val fu = RUser.as("fu") - findItemsBase(q, 0).unwrap + findItemsBase(q.fix, 0).unwrap .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num"))) .changeFrom(_.innerJoin(fu, fu.uid === f.owner)) - .changeWhere(c => c && queryCondition(q)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(f.id, f.name, f.owner, fu.login) .build .query[FolderCount] .to[List] } - def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = { + def searchFieldSummary(today: LocalDate)(q: Query): ConnectionIO[List[FieldStats]] = { val fieldJoin = from(cv) .innerJoin(cf, cf.id === cv.field) .innerJoin(i, i.id === cv.itemId) val base = - findItemsBase(q, 0).unwrap + findItemsBase(q.fix, 0).unwrap .changeFrom(_.prepend(fieldJoin)) - .changeWhere(c => c && queryCondition(q)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(GroupBy(cf.all)) val basicFields = Nel.of( @@ -374,7 +386,7 @@ object QItem { ) ) - val from = findItemsBase(q, maxNoteLen) + val from = findItemsBase(q.fix, maxNoteLen) .appendCte(cte) .appendSelect(Tids.weight.s) .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id)) @@ -490,6 +502,16 @@ object QItem { collective: Ident, excludeFileMeta: Set[Ident] ): ConnectionIO[Vector[RItem]] = { + val qq = findByChecksumQuery(checksum, collective, excludeFileMeta).build + logger.debug(s"FindByChecksum: $qq") + qq.query[RItem].to[Vector] + } + + def findByChecksumQuery( + checksum: String, + collective: Ident, + excludeFileMeta: Set[Ident] + ): Select = { val m1 = RFileMeta.as("m1") val m2 = RFileMeta.as("m2") val m3 = RFileMeta.as("m3") @@ -498,26 +520,23 @@ object QItem { val s = RAttachmentSource.as("s") val r = RAttachmentArchive.as("r") val fms = Nel.of(m1, m2, m3) - val qq = - Select( - select(i.all), - from(i) - .innerJoin(a, a.itemId === i.id) - .innerJoin(s, s.id === a.id) - .innerJoin(m1, m1.id === a.fileId) - .innerJoin(m2, m2.id === s.fileId) - .leftJoin(r, r.id === a.id) - .leftJoin(m3, m3.id === r.fileId), - where( - i.cid === collective && - Condition.Or(fms.map(m => m.checksum === checksum)) &&? - Nel - .fromList(excludeFileMeta.toList) - .map(excl => Condition.And(fms.map(m => m.id.isNull || m.id.notIn(excl)))) - ) - ).distinct.build - logger.debug(s"FindByChecksum: $qq") - qq.query[RItem].to[Vector] + Select( + select(i.all), + from(i) + .innerJoin(a, a.itemId === i.id) + .innerJoin(s, s.id === a.id) + .innerJoin(m1, m1.id === a.fileId) + .innerJoin(m2, m2.id === s.fileId) + .leftJoin(r, r.id === a.id) + .leftJoin(m3, m3.id === r.fileId), + where( + i.cid === collective && + Condition.Or(fms.map(m => m.checksum === checksum)) &&? + Nel + .fromList(excludeFileMeta.toList) + .map(excl => Condition.And(fms.map(m => m.id.isNull || m.id.notIn(excl)))) + ) + ).distinct } final case class NameAndNotes( diff --git a/modules/store/src/main/scala/docspell/store/queries/Query.scala b/modules/store/src/main/scala/docspell/store/queries/Query.scala index 0d68bdef..6be04a1e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/Query.scala +++ b/modules/store/src/main/scala/docspell/store/queries/Query.scala @@ -1,57 +1,104 @@ package docspell.store.queries import docspell.common._ +import docspell.query.ItemQuery +import docspell.store.qb.Column import docspell.store.records.RItem -case class Query( - account: AccountId, - name: Option[String], - states: Seq[ItemState], - direction: Option[Direction], - corrPerson: Option[Ident], - corrOrg: Option[Ident], - concPerson: Option[Ident], - concEquip: Option[Ident], - folder: Option[Ident], - tagsInclude: List[Ident], - tagsExclude: List[Ident], - tagCategoryIncl: List[String], - tagCategoryExcl: List[String], - dateFrom: Option[Timestamp], - dateTo: Option[Timestamp], - dueDateFrom: Option[Timestamp], - dueDateTo: Option[Timestamp], - allNames: Option[String], - itemIds: Option[Set[Ident]], - customValues: Seq[CustomValue], - source: Option[String], - orderAsc: Option[RItem.Table => docspell.store.qb.Column[_]] -) +case class Query(fix: Query.Fix, cond: Query.QueryCond) { + def withCond(f: Query.QueryCond => Query.QueryCond): Query = + copy(cond = f(cond)) + + def withOrder(orderAsc: RItem.Table => Column[_]): Query = + withFix(_.copy(orderAsc = Some(orderAsc))) + + def withFix(f: Query.Fix => Query.Fix): Query = + copy(fix = f(fix)) + + def isEmpty: Boolean = + fix.isEmpty && cond.isEmpty + + def nonEmpty: Boolean = + !isEmpty +} object Query { + + case class Fix( + account: AccountId, + itemIds: Option[Set[Ident]], + orderAsc: Option[RItem.Table => Column[_]] + ) { + + def isEmpty: Boolean = + itemIds.isEmpty + } + + sealed trait QueryCond { + def isEmpty: Boolean + + def nonEmpty: Boolean = + !isEmpty + } + + case class QueryForm( + name: Option[String], + states: Seq[ItemState], + direction: Option[Direction], + corrPerson: Option[Ident], + corrOrg: Option[Ident], + concPerson: Option[Ident], + concEquip: Option[Ident], + folder: Option[Ident], + tagsInclude: List[Ident], + tagsExclude: List[Ident], + tagCategoryIncl: List[String], + tagCategoryExcl: List[String], + dateFrom: Option[Timestamp], + dateTo: Option[Timestamp], + dueDateFrom: Option[Timestamp], + dueDateTo: Option[Timestamp], + allNames: Option[String], + itemIds: Option[Set[Ident]], + customValues: Seq[CustomValue], + source: Option[String] + ) extends QueryCond { + + def isEmpty: Boolean = + this == QueryForm.empty + } + object QueryForm { + val empty = + QueryForm( + None, + Seq.empty, + None, + None, + None, + None, + None, + None, + Nil, + Nil, + Nil, + Nil, + None, + None, + None, + None, + None, + None, + Seq.empty, + None + ) + } + + case class QueryExpr(q: ItemQuery) extends QueryCond { + def isEmpty: Boolean = + q.expr == ItemQuery.all.expr + } + def empty(account: AccountId): Query = - Query( - account, - None, - Seq.empty, - None, - None, - None, - None, - None, - None, - Nil, - Nil, - Nil, - Nil, - None, - None, - None, - None, - None, - None, - Seq.empty, - None, - None - ) + Query(Fix(account, None, None), QueryForm.empty) + } diff --git a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala index 5ab70d20..16a2f1bb 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QueryWildcard.scala @@ -19,4 +19,11 @@ object QueryWildcard { else res } + def atEnd(s: String): String = + if (s.endsWith("*")) s"${s.dropRight(1)}%" + else s + + def addAtEnd(s: String): String = + if (s.endsWith("*")) atEnd(s) + else s"${s}%" } diff --git a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala index 71d261bf..012dad4c 100644 --- a/modules/store/src/main/scala/docspell/store/records/TagItemName.scala +++ b/modules/store/src/main/scala/docspell/store/records/TagItemName.scala @@ -42,9 +42,27 @@ object TagItemName { def itemsWithEitherTag(tags: NonEmptyList[Ident]): Select = Select(ti.itemId.s, from(ti), orTags(tags)).distinct + def itemsWithEitherTagNameOrIds(tags: NonEmptyList[String]): Select = + Select( + ti.itemId.s, + from(ti).innerJoin(t, t.tid === ti.tagId), + ti.tagId.cast[String].in(tags) || t.name.inLower(tags.map(_.toLowerCase)) + ).distinct + def itemsWithAllTags(tags: NonEmptyList[Ident]): Select = intersect(tags.map(tid => Select(ti.itemId.s, from(ti), ti.tagId === tid).distinct)) + def itemsWithAllTagNameOrIds(tags: NonEmptyList[String]): Select = + intersect( + tags.map(tag => + Select( + ti.itemId.s, + from(ti).innerJoin(t, t.tid === ti.tagId), + ti.tagId ==== tag || t.name.lowerEq(tag.toLowerCase) + ).distinct + ) + ) + def itemsWithEitherTagOrCategory( tags: NonEmptyList[Ident], cats: NonEmptyList[String] diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala new file mode 100644 index 00000000..839eb242 --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -0,0 +1,67 @@ +package docspell.store + +import cats.effect._ +import docspell.common.LenientUri +import docspell.store.impl.StoreImpl +import doobie._ +import org.h2.jdbcx.JdbcConnectionPool + +import scala.concurrent.ExecutionContext + +trait StoreFixture { + def withStore(db: String)(code: Store[IO] => IO[Unit]): Unit = { + //StoreFixture.store(StoreFixture.memoryDB(db)).use(code).unsafeRunSync() + val jdbc = StoreFixture.memoryDB(db) + val xa = StoreFixture.globalXA(jdbc) + val store = new StoreImpl[IO](jdbc, xa) + store.migrate.unsafeRunSync() + code(store).unsafeRunSync() + } + + def withXA(db: String)(code: Transactor[IO] => IO[Unit]): Unit = + StoreFixture.makeXA(StoreFixture.memoryDB(db)).use(code).unsafeRunSync() + +} + +object StoreFixture { + implicit def contextShift: ContextShift[IO] = + IO.contextShift(ExecutionContext.global) + + def memoryDB(dbname: String): JdbcConfig = + JdbcConfig( + LenientUri.unsafe( + s"jdbc:h2:mem:$dbname;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1" + ), + "sa", + "" + ) + + def globalXA(jdbc: JdbcConfig): Transactor[IO] = + Transactor.fromDriverManager( + "org.h2.Driver", + jdbc.url.asString, + jdbc.user, + jdbc.password + ) + + def makeXA(jdbc: JdbcConfig): Resource[IO, Transactor[IO]] = { + def jdbcConnPool = + JdbcConnectionPool.create(jdbc.url.asString, jdbc.user, jdbc.password) + + val makePool = Resource.make(IO(jdbcConnPool))(cp => IO(cp.dispose())) + + for { + ec <- ExecutionContexts.cachedThreadPool[IO] + blocker <- Blocker[IO] + pool <- makePool + xa = Transactor.fromDataSource[IO].apply(pool, ec, blocker) + } yield xa + } + + def store(jdbc: JdbcConfig): Resource[IO, Store[IO]] = + for { + xa <- makeXA(jdbc) + store = new StoreImpl[IO](jdbc, xa) + _ <- Resource.liftF(store.migrate) + } yield store +} diff --git a/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala b/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala new file mode 100644 index 00000000..ebc8fd33 --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/generator/ItemQueryGeneratorTest.scala @@ -0,0 +1,45 @@ +package docspell.store.generator + +import java.time.LocalDate + +import docspell.store.records._ +import minitest._ +import docspell.common._ +import docspell.query.ItemQueryParser +import docspell.store.queries.AttachCountTable +import docspell.store.qb.DSL._ +import docspell.store.qb.generator.{ItemQueryGenerator, Tables} + +object ItemQueryGeneratorTest extends SimpleTestSuite { + import docspell.store.impl.DoobieMeta._ + + val tables = Tables( + RItem.as("i"), + ROrganization.as("co"), + RPerson.as("cp"), + RPerson.as("np"), + REquipment.as("ne"), + RFolder.as("f"), + RAttachment.as("a"), + RAttachmentMeta.as("m"), + AttachCountTable("cta") + ) + val now: LocalDate = LocalDate.of(2021, 2, 25) + + def mkTimestamp(year: Int, month: Int, day: Int): Timestamp = + Timestamp.atUtc(LocalDate.of(year, month, day).atStartOfDay()) + + test("basic test") { + val q = ItemQueryParser + .parseUnsafe("(& name:hello date>=2020-02-01 (| source:expense* folder=test ))") + val cond = ItemQueryGenerator(now, tables, Ident.unsafe("coll"))(q) + val expect = + tables.item.name.like("hello") && + coalesce(tables.item.itemDate.s, tables.item.created.s) >= + mkTimestamp(2020, 2, 1) && + (tables.item.source.like("expense%") || tables.folder.name === "test") + + assertEquals(cond, expect) + } + +} diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index efc1d551..30202fd6 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -156,10 +156,10 @@ import Api.Model.ImapSettings exposing (ImapSettings) import Api.Model.ImapSettingsList exposing (ImapSettingsList) import Api.Model.InviteResult exposing (InviteResult) import Api.Model.ItemDetail exposing (ItemDetail) -import Api.Model.ItemFtsSearch exposing (ItemFtsSearch) import Api.Model.ItemInsights exposing (ItemInsights) import Api.Model.ItemLightList exposing (ItemLightList) import Api.Model.ItemProposals exposing (ItemProposals) +import Api.Model.ItemQuery exposing (ItemQuery) import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.ItemUploadMeta exposing (ItemUploadMeta) import Api.Model.ItemsAndDate exposing (ItemsAndDate) @@ -1684,34 +1684,34 @@ moveAttachmentBefore flags itemId data receive = itemIndexSearch : Flags - -> ItemFtsSearch + -> ItemQuery -> (Result Http.Error ItemLightList -> msg) -> Cmd msg itemIndexSearch flags query receive = Http2.authPost { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchIndex" , account = getAccount flags - , body = Http.jsonBody (Api.Model.ItemFtsSearch.encode query) + , body = Http.jsonBody (Api.Model.ItemQuery.encode query) , expect = Http.expectJson receive Api.Model.ItemLightList.decoder } -itemSearch : Flags -> ItemSearch -> (Result Http.Error ItemLightList -> msg) -> Cmd msg +itemSearch : Flags -> ItemQuery -> (Result Http.Error ItemLightList -> msg) -> Cmd msg itemSearch flags search receive = Http2.authPost - { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchWithTags" + { url = flags.config.baseUrl ++ "/api/v1/sec/item/search" , account = getAccount flags - , body = Http.jsonBody (Api.Model.ItemSearch.encode search) + , body = Http.jsonBody (Api.Model.ItemQuery.encode search) , expect = Http.expectJson receive Api.Model.ItemLightList.decoder } -itemSearchStats : Flags -> ItemSearch -> (Result Http.Error SearchStats -> msg) -> Cmd msg +itemSearchStats : Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg itemSearchStats flags search receive = Http2.authPost { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchStats" , account = getAccount flags - , body = Http.jsonBody (Api.Model.ItemSearch.encode search) + , body = Http.jsonBody (Api.Model.ItemQuery.encode search) , expect = Http.expectJson receive Api.Model.SearchStats.decoder } diff --git a/modules/webapp/src/main/elm/Comp/PowerSearchInput.elm b/modules/webapp/src/main/elm/Comp/PowerSearchInput.elm new file mode 100644 index 00000000..2a868fa0 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/PowerSearchInput.elm @@ -0,0 +1,177 @@ +module Comp.PowerSearchInput exposing + ( Action(..) + , Model + , Msg + , init + , update + , viewInput + , viewResult + ) + +import Data.DropdownStyle +import Data.QueryParseResult exposing (QueryParseResult) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Ports +import Styles as S +import Throttle exposing (Throttle) +import Time +import Util.Html exposing (KeyCode(..)) +import Util.Maybe + + +type alias Model = + { input : Maybe String + , result : QueryParseResult + , parseThrottle : Throttle Msg + } + + +init : Model +init = + { input = Nothing + , result = Data.QueryParseResult.success + , parseThrottle = Throttle.create 1 + } + + +type Msg + = SetSearch String + | KeyUpMsg (Maybe KeyCode) + | ParseResultMsg QueryParseResult + | UpdateThrottle + + +type Action + = NoAction + | SubmitSearch + + +type alias Result = + { model : Model + , cmd : Cmd Msg + , action : Action + , subs : Sub Msg + } + + + +--- Update + + +update : Msg -> Model -> Result +update msg model = + case msg of + SetSearch str -> + let + parseCmd = + Ports.checkSearchQueryString str + + parseSub = + Ports.receiveCheckQueryResult ParseResultMsg + + ( newThrottle, cmd ) = + Throttle.try parseCmd model.parseThrottle + + model_ = + { model + | input = Util.Maybe.fromString str + , parseThrottle = newThrottle + , result = + if str == "" then + Data.QueryParseResult.success + + else + model.result + } + in + { model = model_ + , cmd = cmd + , action = NoAction + , subs = Sub.batch [ throttleUpdate model_, parseSub ] + } + + KeyUpMsg (Just Enter) -> + Result model Cmd.none SubmitSearch Sub.none + + KeyUpMsg _ -> + let + parseSub = + Ports.receiveCheckQueryResult ParseResultMsg + in + Result model Cmd.none NoAction (Sub.batch [ throttleUpdate model, parseSub ]) + + ParseResultMsg lm -> + Result { model | result = lm } Cmd.none NoAction Sub.none + + UpdateThrottle -> + let + parseSub = + Ports.receiveCheckQueryResult ParseResultMsg + + ( newThrottle, cmd ) = + Throttle.update model.parseThrottle + + model_ = + { model | parseThrottle = newThrottle } + in + { model = model_ + , cmd = cmd + , action = NoAction + , subs = Sub.batch [ throttleUpdate model_, parseSub ] + } + + +throttleUpdate : Model -> Sub Msg +throttleUpdate model = + Throttle.ifNeeded + (Time.every 100 (\_ -> UpdateThrottle)) + model.parseThrottle + + + +--- View + + +viewInput : List (Attribute Msg) -> Model -> Html Msg +viewInput attrs model = + input + (attrs + ++ [ type_ "text" + , placeholder "Search query …" + , onInput SetSearch + , Util.Html.onKeyUpCode KeyUpMsg + , Maybe.map value model.input + |> Maybe.withDefault (value "") + , class S.textInput + , class "text-sm " + ] + ) + [] + + +viewResult : List ( String, Bool ) -> Model -> Html Msg +viewResult classes model = + div + [ classList [ ( "hidden", model.result.success ) ] + , classList classes + , class resultStyle + ] + [ p [ class "font-mono text-sm" ] + [ text model.result.input + ] + , pre [ class "font-mono text-sm" ] + [ List.repeat model.result.index " " + |> String.join "" + |> text + , text "^" + ] + , ul [] + (List.map (\line -> li [] [ text line ]) model.result.messages) + ] + + +resultStyle : String +resultStyle = + S.warnMessageColors ++ " absolute left-0 max-h-44 w-full overflow-y-auto z-50 shadow-lg transition duration-200 top-9 border-0 border-b border-l border-r rounded-b px-2 py-2" diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index 5a88fee6..669d71b7 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -3,7 +3,7 @@ module Comp.SearchMenu exposing , Msg(..) , NextState , TextSearchModel - , getItemSearch + , getItemQuery , init , isFulltextSearch , isNamesSearch @@ -21,7 +21,7 @@ import Api.Model.EquipmentList exposing (EquipmentList) import Api.Model.FolderStats exposing (FolderStats) import Api.Model.IdName exposing (IdName) import Api.Model.ItemFieldValue exposing (ItemFieldValue) -import Api.Model.ItemSearch exposing (ItemSearch) +import Api.Model.ItemQuery exposing (ItemQuery) import Api.Model.PersonList exposing (PersonList) import Api.Model.ReferenceList exposing (ReferenceList) import Api.Model.SearchStats exposing (SearchStats) @@ -38,6 +38,7 @@ import Data.DropdownStyle as DS import Data.Fields import Data.Flags exposing (Flags) import Data.Icons as Icons +import Data.ItemQuery as Q exposing (ItemQuery) import Data.PersonUse import Data.UiSettings exposing (UiSettings) import DatePicker exposing (DatePicker) @@ -234,11 +235,21 @@ getDirection model = Nothing -getItemSearch : Model -> ItemSearch -getItemSearch model = +getItemQuery : Model -> Maybe ItemQuery +getItemQuery model = let - e = - Api.Model.ItemSearch.empty + when flag body = + if flag then + Just body + + else + Nothing + + whenNot flag body = + when (not flag) body + + whenNotEmpty list f = + whenNot (List.isEmpty list) (f list) amendWildcards s = if String.startsWith "\"" s && String.endsWith "\"" s then @@ -254,35 +265,52 @@ getItemSearch model = textSearch = textSearchValue model.textSearchModel in - { e - | tagsInclude = model.tagSelection.includeTags |> List.map .tag |> List.map .id - , tagsExclude = model.tagSelection.excludeTags |> List.map .tag |> List.map .id - , corrPerson = Comp.Dropdown.getSelected model.corrPersonModel |> List.map .id |> List.head - , corrOrg = Comp.Dropdown.getSelected model.orgModel |> List.map .id |> List.head - , concPerson = Comp.Dropdown.getSelected model.concPersonModel |> List.map .id |> List.head - , concEquip = Comp.Dropdown.getSelected model.concEquipmentModel |> List.map .id |> List.head - , folder = model.selectedFolder |> Maybe.map .id - , direction = - Comp.Dropdown.getSelected model.directionModel - |> List.head - |> Maybe.map Data.Direction.toString - , inbox = model.inboxCheckbox - , dateFrom = model.fromDate - , dateUntil = model.untilDate - , dueDateFrom = model.fromDueDate - , dueDateUntil = model.untilDueDate - , name = - model.nameModel - |> Maybe.map amendWildcards - , allNames = - textSearch.nameSearch - |> Maybe.map amendWildcards - , fullText = textSearch.fullText - , tagCategoriesInclude = model.tagSelection.includeCats |> List.map .name - , tagCategoriesExclude = model.tagSelection.excludeCats |> List.map .name - , customValues = Data.CustomFieldChange.toFieldValues model.customValues - , source = model.sourceModel - } + Q.and + [ when model.inboxCheckbox (Q.Inbox True) + , whenNotEmpty (model.tagSelection.includeTags |> List.map (.tag >> .id)) + (Q.TagIds Q.AllMatch) + , whenNotEmpty (model.tagSelection.excludeTags |> List.map (.tag >> .id)) + (\ids -> Q.Not (Q.TagIds Q.AnyMatch ids)) + , whenNotEmpty (model.tagSelection.includeCats |> List.map .name) + (Q.CatNames Q.AllMatch) + , whenNotEmpty (model.tagSelection.excludeCats |> List.map .name) + (\ids -> Q.Not <| Q.CatNames Q.AnyMatch ids) + , model.selectedFolder |> Maybe.map .id |> Maybe.map (Q.FolderId Q.Eq) + , Comp.Dropdown.getSelected model.orgModel + |> List.map .id + |> List.head + |> Maybe.map (Q.CorrOrgId Q.Eq) + , Comp.Dropdown.getSelected model.corrPersonModel + |> List.map .id + |> List.head + |> Maybe.map (Q.CorrPersId Q.Eq) + , Comp.Dropdown.getSelected model.concPersonModel + |> List.map .id + |> List.head + |> Maybe.map (Q.ConcPersId Q.Eq) + , Comp.Dropdown.getSelected model.concEquipmentModel + |> List.map .id + |> List.head + |> Maybe.map (Q.ConcEquipId Q.Eq) + , whenNotEmpty (Data.CustomFieldChange.toFieldValues model.customValues) + (List.map (Q.CustomFieldId Q.Like) >> Q.And) + , Maybe.map (Q.DateMs Q.Gte) model.fromDate + , Maybe.map (Q.DateMs Q.Lte) model.untilDate + , Maybe.map (Q.DueDateMs Q.Gte) model.fromDueDate + , Maybe.map (Q.DueDateMs Q.Lte) model.untilDueDate + , Maybe.map (Q.Source Q.Like) model.sourceModel + , model.nameModel + |> Maybe.map amendWildcards + |> Maybe.map (Q.ItemName Q.Like) + , textSearch.nameSearch + |> Maybe.map amendWildcards + |> Maybe.map Q.AllNames + , Comp.Dropdown.getSelected model.directionModel + |> List.head + |> Maybe.map Q.Dir + , textSearch.fullText + |> Maybe.map Q.Contents + ] resetModel : Model -> Model @@ -437,7 +465,7 @@ updateDrop ddm flags settings msg model = { model = mdp , cmd = Cmd.batch - [ Api.itemSearchStats flags Api.Model.ItemSearch.empty GetAllTagsResp + [ Api.itemSearchStats flags Api.Model.ItemQuery.empty GetAllTagsResp , Api.getOrgLight flags GetOrgResp , Api.getEquipments flags "" GetEquipResp , Api.getPersons flags "" GetPersonResp @@ -450,7 +478,7 @@ updateDrop ddm flags settings msg model = ResetForm -> { model = resetModel model - , cmd = Api.itemSearchStats flags Api.Model.ItemSearch.empty GetAllTagsResp + , cmd = Api.itemSearchStats flags Api.Model.ItemQuery.empty GetAllTagsResp , stateChange = True , dragDrop = DD.DragDropData ddm Nothing } diff --git a/modules/webapp/src/main/elm/Comp/UiSettingsForm.elm b/modules/webapp/src/main/elm/Comp/UiSettingsForm.elm index 9594c7e5..14cbc18c 100644 --- a/modules/webapp/src/main/elm/Comp/UiSettingsForm.elm +++ b/modules/webapp/src/main/elm/Comp/UiSettingsForm.elm @@ -58,6 +58,7 @@ type alias Model = , showPatternHelp : Bool , searchStatsVisible : Bool , sideMenuVisible : Bool + , powerSearchEnabled : Bool , openTabs : Set String } @@ -151,6 +152,7 @@ init flags settings = , showPatternHelp = False , searchStatsVisible = settings.searchStatsVisible , sideMenuVisible = settings.sideMenuVisible + , powerSearchEnabled = settings.powerSearchEnabled , openTabs = Set.empty } , Api.getTags flags "" GetTagsResp @@ -178,6 +180,7 @@ type Msg | ToggleSearchStatsVisible | ToggleAkkordionTab String | ToggleSideMenuVisible + | TogglePowerSearch @@ -460,6 +463,15 @@ update sett msg model = , Just { sett | sideMenuVisible = next } ) + TogglePowerSearch -> + let + next = + not model.powerSearchEnabled + in + ( { model | powerSearchEnabled = next } + , Just { sett | powerSearchEnabled = next } + ) + --- View @@ -763,6 +775,15 @@ settingFormTabs flags _ model = , label = "Show basic search statistics by default" } ] + , div [ class "mb-4" ] + [ MB.viewItem <| + MB.Checkbox + { id = "uisetting-powersearch-enabled" + , value = model.powerSearchEnabled + , tagger = \_ -> TogglePowerSearch + , label = "Enable power-user search bar" + } + ] ] } , { title = "Item Cards" diff --git a/modules/webapp/src/main/elm/Data/CustomFieldChange.elm b/modules/webapp/src/main/elm/Data/CustomFieldChange.elm index 04f69b1c..2ee75c02 100644 --- a/modules/webapp/src/main/elm/Data/CustomFieldChange.elm +++ b/modules/webapp/src/main/elm/Data/CustomFieldChange.elm @@ -10,7 +10,6 @@ module Data.CustomFieldChange exposing import Api.Model.CustomField exposing (CustomField) import Api.Model.CustomFieldValue exposing (CustomFieldValue) -import Api.Model.ItemFieldValue exposing (ItemFieldValue) import Dict exposing (Dict) diff --git a/modules/webapp/src/main/elm/Data/ItemQuery.elm b/modules/webapp/src/main/elm/Data/ItemQuery.elm new file mode 100644 index 00000000..f301abe6 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/ItemQuery.elm @@ -0,0 +1,210 @@ +module Data.ItemQuery exposing + ( AttrMatch(..) + , ItemQuery(..) + , TagMatch(..) + , and + , render + , renderMaybe + , request + ) + +{-| Models the query language for the purpose of generating a query string. +-} + +import Api.Model.CustomFieldValue exposing (CustomFieldValue) +import Api.Model.ItemQuery as RQ +import Data.Direction exposing (Direction) + + +type TagMatch + = AnyMatch + | AllMatch + + +type AttrMatch + = Eq + | Neq + | Lt + | Gt + | Lte + | Gte + | Like + + +type ItemQuery + = Inbox Bool + | And (List ItemQuery) + | Or (List ItemQuery) + | Not ItemQuery + | TagIds TagMatch (List String) + | CatNames TagMatch (List String) + | FolderId AttrMatch String + | CorrOrgId AttrMatch String + | CorrPersId AttrMatch String + | ConcPersId AttrMatch String + | ConcEquipId AttrMatch String + | CustomField AttrMatch CustomFieldValue + | CustomFieldId AttrMatch CustomFieldValue + | DateMs AttrMatch Int + | DueDateMs AttrMatch Int + | Source AttrMatch String + | Dir Direction + | ItemIdIn (List String) + | ItemName AttrMatch String + | AllNames String + | Contents String + | Fragment String + + +and : List (Maybe ItemQuery) -> Maybe ItemQuery +and list = + case List.filterMap identity list of + [] -> + Nothing + + es -> + Just (And es) + + +request : Maybe ItemQuery -> RQ.ItemQuery +request mq = + { offset = Nothing + , limit = Nothing + , withDetails = Just True + , query = renderMaybe mq + } + + +renderMaybe : Maybe ItemQuery -> String +renderMaybe mq = + Maybe.map render mq + |> Maybe.withDefault "" + + +render : ItemQuery -> String +render q = + let + boolStr flag = + if flag then + "yes" + + else + "no" + + between left right str = + left ++ str ++ right + + surround lr str = + between lr lr str + + tagMatchStr tm = + case tm of + AnyMatch -> + ":" + + AllMatch -> + "=" + + quoteStr = + String.replace "\"" "\\\"" + >> surround "\"" + in + case q of + And inner -> + List.map render inner + |> String.join " " + |> between "(& " " )" + + Or inner -> + List.map render inner + |> String.join " " + |> between "(| " " )" + + Not inner -> + "!" ++ render inner + + Inbox flag -> + "inbox:" ++ boolStr flag + + TagIds m ids -> + List.map quoteStr ids + |> String.join "," + |> between ("tag.id" ++ tagMatchStr m) "" + + CatNames m ids -> + List.map quoteStr ids + |> String.join "," + |> between ("cat" ++ tagMatchStr m) "" + + FolderId m id -> + "folder.id" ++ attrMatch m ++ quoteStr id + + CorrOrgId m id -> + "corr.org.id" ++ attrMatch m ++ quoteStr id + + CorrPersId m id -> + "corr.pers.id" ++ attrMatch m ++ quoteStr id + + ConcPersId m id -> + "conc.pers.id" ++ attrMatch m ++ quoteStr id + + ConcEquipId m id -> + "conc.equip.id" ++ attrMatch m ++ quoteStr id + + CustomField m kv -> + "f:" ++ kv.field ++ attrMatch m ++ quoteStr kv.value + + CustomFieldId m kv -> + "f.id:" ++ kv.field ++ attrMatch m ++ quoteStr kv.value + + DateMs m ms -> + "date" ++ attrMatch m ++ "ms" ++ String.fromInt ms + + DueDateMs m ms -> + "due" ++ attrMatch m ++ "ms" ++ String.fromInt ms + + Source m str -> + "source" ++ attrMatch m ++ quoteStr str + + Dir dir -> + "incoming:" ++ boolStr (dir == Data.Direction.Incoming) + + ItemIdIn ids -> + "id~=" ++ String.join "," ids + + ItemName m str -> + "name" ++ attrMatch m ++ quoteStr str + + AllNames str -> + "$names:" ++ quoteStr str + + Contents str -> + "content:" ++ quoteStr str + + Fragment str -> + "(& " ++ str ++ " )" + + +attrMatch : AttrMatch -> String +attrMatch am = + case am of + Eq -> + "=" + + Neq -> + "!=" + + Like -> + ":" + + Gt -> + ">" + + Gte -> + ">=" + + Lt -> + "<" + + Lte -> + "<=" diff --git a/modules/webapp/src/main/elm/Data/QueryParseResult.elm b/modules/webapp/src/main/elm/Data/QueryParseResult.elm new file mode 100644 index 00000000..bb0046ba --- /dev/null +++ b/modules/webapp/src/main/elm/Data/QueryParseResult.elm @@ -0,0 +1,14 @@ +module Data.QueryParseResult exposing (QueryParseResult, success) + + +type alias QueryParseResult = + { success : Bool + , input : String + , index : Int + , messages : List String + } + + +success : QueryParseResult +success = + QueryParseResult True "" 0 [] diff --git a/modules/webapp/src/main/elm/Data/UiSettings.elm b/modules/webapp/src/main/elm/Data/UiSettings.elm index 0153b63c..988eb2d4 100644 --- a/modules/webapp/src/main/elm/Data/UiSettings.elm +++ b/modules/webapp/src/main/elm/Data/UiSettings.elm @@ -62,6 +62,7 @@ type alias StoredUiSettings = , cardPreviewFullWidth : Bool , uiTheme : Maybe String , sideMenuVisible : Bool + , powerSearchEnabled : Bool } @@ -92,6 +93,7 @@ type alias UiSettings = , cardPreviewFullWidth : Bool , uiTheme : UiTheme , sideMenuVisible : Bool + , powerSearchEnabled : Bool } @@ -162,6 +164,7 @@ defaults = , cardPreviewFullWidth = False , uiTheme = Data.UiTheme.Light , sideMenuVisible = True + , powerSearchEnabled = False } @@ -213,6 +216,7 @@ merge given fallback = Maybe.andThen Data.UiTheme.fromString given.uiTheme |> Maybe.withDefault fallback.uiTheme , sideMenuVisible = given.sideMenuVisible + , powerSearchEnabled = given.powerSearchEnabled } @@ -249,6 +253,7 @@ toStoredUiSettings settings = , cardPreviewFullWidth = settings.cardPreviewFullWidth , uiTheme = Just (Data.UiTheme.toString settings.uiTheme) , sideMenuVisible = settings.sideMenuVisible + , powerSearchEnabled = settings.powerSearchEnabled } diff --git a/modules/webapp/src/main/elm/Page/Home/Data.elm b/modules/webapp/src/main/elm/Page/Home/Data.elm index b0bf156c..7e6246cb 100644 --- a/modules/webapp/src/main/elm/Page/Home/Data.elm +++ b/modules/webapp/src/main/elm/Page/Home/Data.elm @@ -27,10 +27,12 @@ import Comp.ItemCardList import Comp.ItemDetail.FormChange exposing (FormChange) import Comp.ItemDetail.MultiEditMenu exposing (SaveNameState(..)) import Comp.LinkTarget exposing (LinkTarget) +import Comp.PowerSearchInput import Comp.SearchMenu import Comp.YesNoDimmer import Data.Flags exposing (Flags) import Data.ItemNav exposing (ItemNav) +import Data.ItemQuery as Q import Data.Items import Data.UiSettings exposing (UiSettings) import Http @@ -55,6 +57,7 @@ type alias Model = , dragDropData : DD.DragDropData , scrollToCard : Maybe String , searchStats : SearchStats + , powerSearchInput : Comp.PowerSearchInput.Model } @@ -120,6 +123,7 @@ init flags viewMode = , scrollToCard = Nothing , viewMode = viewMode , searchStats = Api.Model.SearchStats.empty + , powerSearchInput = Comp.PowerSearchInput.init } @@ -193,6 +197,8 @@ type Msg | SetLinkTarget LinkTarget | SearchStatsResp (Result Http.Error SearchStats) | TogglePreviewFullWidth + | PowerSearchMsg Comp.PowerSearchInput.Msg + | KeyUpPowerSearchbarMsg (Maybe KeyCode) type SearchType @@ -239,12 +245,16 @@ doSearchDefaultCmd : SearchParam -> Model -> Cmd Msg doSearchDefaultCmd param model = let smask = - Comp.SearchMenu.getItemSearch model.searchMenuModel + Q.request <| + Q.and + [ Comp.SearchMenu.getItemQuery model.searchMenuModel + , Maybe.map Q.Fragment model.powerSearchInput.input + ] mask = { smask - | limit = param.pageSize - , offset = param.offset + | limit = Just param.pageSize + , offset = Just param.offset } in if param.offset == 0 then diff --git a/modules/webapp/src/main/elm/Page/Home/Update.elm b/modules/webapp/src/main/elm/Page/Home/Update.elm index a2cddb7f..9d8efee9 100644 --- a/modules/webapp/src/main/elm/Page/Home/Update.elm +++ b/modules/webapp/src/main/elm/Page/Home/Update.elm @@ -1,18 +1,18 @@ module Page.Home.Update exposing (update) import Api -import Api.Model.IdList exposing (IdList) import Api.Model.ItemLightList exposing (ItemLightList) -import Api.Model.ItemSearch import Browser.Navigation as Nav import Comp.FixedDropdown import Comp.ItemCardList import Comp.ItemDetail.FormChange exposing (FormChange(..)) import Comp.ItemDetail.MultiEditMenu exposing (SaveNameState(..)) import Comp.LinkTarget exposing (LinkTarget) +import Comp.PowerSearchInput import Comp.SearchMenu import Comp.YesNoDimmer import Data.Flags exposing (Flags) +import Data.ItemQuery as Q import Data.ItemSelection import Data.Items import Data.UiSettings exposing (UiSettings) @@ -53,7 +53,7 @@ update mId key flags settings msg model = ResetSearch -> let nm = - { model | searchOffset = 0 } + { model | searchOffset = 0, powerSearchInput = Comp.PowerSearchInput.init } in update mId key flags settings (SearchMenuMsg Comp.SearchMenu.ResetForm) nm @@ -579,6 +579,30 @@ update mId key flags settings msg model = in noSub ( model, cmd ) + PowerSearchMsg lm -> + let + result = + Comp.PowerSearchInput.update lm model.powerSearchInput + + cmd_ = + Cmd.map PowerSearchMsg result.cmd + + model_ = + { model | powerSearchInput = result.model } + in + case result.action of + Comp.PowerSearchInput.NoAction -> + ( model_, cmd_, Sub.map PowerSearchMsg result.subs ) + + Comp.PowerSearchInput.SubmitSearch -> + update mId key flags settings (DoSearch model_.searchTypeDropdownValue) model_ + + KeyUpPowerSearchbarMsg (Just Enter) -> + update mId key flags settings (DoSearch model.searchTypeDropdownValue) model + + KeyUpPowerSearchbarMsg _ -> + withSub ( model, Cmd.none ) + --- Helpers @@ -648,16 +672,15 @@ loadChangedItems flags ids = else let - searchInit = - Api.Model.ItemSearch.empty - idList = - IdList (Set.toList ids) + Set.toList ids + + searchInit = + Q.request (Just <| Q.ItemIdIn idList) search = { searchInit - | itemSubset = Just idList - , limit = Set.size ids + | limit = Just <| Set.size ids } in Api.itemSearch flags search ReplaceChangedItemsResp diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index f4d8124d..c12a31c9 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -320,8 +320,9 @@ viewSearchBar flags model = [ a [ classList [ ( "search-menu-toggle ui icon button", True ) - , ( "primary", not (searchMenuFilled model) ) - , ( "secondary", searchMenuFilled model ) + + -- , ( "primary", not (searchMenuFilled model) ) + -- , ( "secondary", searchMenuFilled model ) ] , onClick ToggleSearchMenu , href "#" @@ -332,24 +333,23 @@ viewSearchBar flags model = , div [ class "right menu" ] [ div [ class "fitted item" ] [ div [ class "ui left icon right action input" ] - [ i - [ classList - [ ( "search link icon", not model.searchInProgress ) - , ( "loading spinner icon", model.searchInProgress ) - ] - , href "#" - , onClick (DoSearch model.searchTypeDropdownValue) - ] - (if hasMoreSearch model then - [ i [ class "icons search-corner-icons" ] - [ i [ class "tiny blue circle icon" ] [] - ] - ] - - else - [] - ) - , input + [ -- i + -- [ classList + -- [ ( "search link icon", not model.searchInProgress ) + -- , ( "loading spinner icon", model.searchInProgress ) + -- ] + -- , href "#" + -- , onClick (DoSearch model.searchTypeDropdownValue) + -- ] + -- (if hasMoreSearch model then + -- [ i [ class "icons search-corner-icons" ] + -- [ i [ class "tiny blue circle icon" ] [] + -- ] + -- ] + -- else + -- [] + -- ) + input [ type_ "text" , placeholder (case model.searchTypeDropdownValue of @@ -384,27 +384,6 @@ viewSearchBar flags model = ] -searchMenuFilled : Model -> Bool -searchMenuFilled model = - let - is = - Comp.SearchMenu.getItemSearch model.searchMenuModel - in - is /= Api.Model.ItemSearch.empty - - -hasMoreSearch : Model -> Bool -hasMoreSearch model = - let - is = - Comp.SearchMenu.getItemSearch model.searchMenuModel - - is_ = - { is | allNames = Nothing, fullText = Nothing } - in - is_ /= Api.Model.ItemSearch.empty - - deleteAllDimmer : Comp.YesNoDimmer.Settings deleteAllDimmer = { message = "Really delete all selected items?" diff --git a/modules/webapp/src/main/elm/Page/Home/View2.elm b/modules/webapp/src/main/elm/Page/Home/View2.elm index ee8c3bdb..4b6d6f8e 100644 --- a/modules/webapp/src/main/elm/Page/Home/View2.elm +++ b/modules/webapp/src/main/elm/Page/Home/View2.elm @@ -3,6 +3,7 @@ module Page.Home.View2 exposing (viewContent, viewSidebar) import Comp.Basic as B import Comp.ItemCardList import Comp.MenuBar as MB +import Comp.PowerSearchInput import Comp.SearchMenu import Comp.SearchStatsView import Comp.YesNoDimmer @@ -92,7 +93,7 @@ itemsBar flags settings model = defaultMenuBar : Flags -> UiSettings -> Model -> Html Msg -defaultMenuBar flags settings model = +defaultMenuBar _ settings model = let btnStyle = S.secondaryBasicButton ++ " text-sm" @@ -100,6 +101,48 @@ defaultMenuBar flags settings model = searchInput = Comp.SearchMenu.textSearchString model.searchMenuModel.textSearchModel + + simpleSearchBar = + div + [ class "relative flex flex-row" ] + [ input + [ type_ "text" + , placeholder + (case model.searchTypeDropdownValue of + ContentOnlySearch -> + "Content search…" + + BasicSearch -> + "Search in names…" + ) + , onInput SetBasicSearch + , Util.Html.onKeyUpCode KeyUpSearchbarMsg + , Maybe.map value searchInput + |> Maybe.withDefault (value "") + , class (String.replace "rounded" "" S.textInput) + , class "py-1 text-sm border-r-0 rounded-l" + ] + [] + , a + [ class S.secondaryBasicButtonPlain + , class "text-sm px-4 py-2 border rounded-r" + , href "#" + , onClick ToggleSearchType + ] + [ i [ class "fa fa-exchange-alt" ] [] + ] + ] + + powerSearchBar = + div + [ class "relative flex flex-grow flex-row" ] + [ Html.map PowerSearchMsg + (Comp.PowerSearchInput.viewInput [] + model.powerSearchInput + ) + , Html.map PowerSearchMsg + (Comp.PowerSearchInput.viewResult [] model.powerSearchInput) + ] in MB.view { end = @@ -129,35 +172,11 @@ defaultMenuBar flags settings model = ] , start = [ MB.CustomElement <| - div - [ class "relative flex flex-row" ] - [ input - [ type_ "text" - , placeholder - (case model.searchTypeDropdownValue of - ContentOnlySearch -> - "Content search…" + if settings.powerSearchEnabled then + powerSearchBar - BasicSearch -> - "Search in names…" - ) - , onInput SetBasicSearch - , Util.Html.onKeyUpCode KeyUpSearchbarMsg - , Maybe.map value searchInput - |> Maybe.withDefault (value "") - , class (String.replace "rounded" "" S.textInput) - , class "py-1 text-sm border-r-0 rounded-l" - ] - [] - , a - [ class S.secondaryBasicButtonPlain - , class "text-sm px-4 py-2 border rounded-r" - , href "#" - , onClick ToggleSearchType - ] - [ i [ class "fa fa-exchange-alt" ] [] - ] - ] + else + simpleSearchBar , MB.CustomButton { tagger = TogglePreviewFullWidth , label = "" @@ -271,7 +290,7 @@ searchStats _ settings model = itemCardList : Flags -> UiSettings -> Model -> List (Html Msg) -itemCardList flags settings model = +itemCardList _ settings model = let itemViewCfg = case model.viewMode of diff --git a/modules/webapp/src/main/elm/Ports.elm b/modules/webapp/src/main/elm/Ports.elm index a8874539..9f630a81 100644 --- a/modules/webapp/src/main/elm/Ports.elm +++ b/modules/webapp/src/main/elm/Ports.elm @@ -1,8 +1,10 @@ port module Ports exposing - ( getUiSettings + ( checkSearchQueryString + , getUiSettings , initClipboard , loadUiSettings , onUiSettingsSaved + , receiveCheckQueryResult , removeAccount , setAccount , setUiTheme @@ -10,7 +12,9 @@ port module Ports exposing ) import Api.Model.AuthResult exposing (AuthResult) +import Api.Model.BasicResult exposing (BasicResult) import Data.Flags exposing (Flags) +import Data.QueryParseResult exposing (QueryParseResult) import Data.UiSettings exposing (StoredUiSettings, UiSettings) import Data.UiTheme exposing (UiTheme) @@ -38,6 +42,12 @@ port uiSettingsSaved : (() -> msg) -> Sub msg port internalSetUiTheme : String -> Cmd msg +port checkSearchQueryString : String -> Cmd msg + + +port receiveCheckQueryResult : (QueryParseResult -> msg) -> Sub msg + + setUiTheme : UiTheme -> Cmd msg setUiTheme theme = internalSetUiTheme (Data.UiTheme.toString theme) diff --git a/modules/webapp/src/main/elm/Styles.elm b/modules/webapp/src/main/elm/Styles.elm index 56a727bc..2d917c49 100644 --- a/modules/webapp/src/main/elm/Styles.elm +++ b/modules/webapp/src/main/elm/Styles.elm @@ -43,7 +43,12 @@ errorMessage = warnMessage : String warnMessage = - " border border-yellow-800 bg-yellow-50 text-yellow-800 dark:border-amber-200 dark:bg-amber-800 dark:text-amber-200 dark:bg-opacity-25 px-2 py-2 rounded " + warnMessageColors ++ " border dark:bg-opacity-25 px-2 py-2 rounded " + + +warnMessageColors : String +warnMessageColors = + " border-yellow-800 bg-yellow-50 text-yellow-800 dark:border-amber-200 dark:bg-amber-800 dark:text-amber-200 " infoMessage : String diff --git a/modules/webapp/src/main/webjar/docspell.js b/modules/webapp/src/main/webjar/docspell.js index 17b1901f..d6fcc4c6 100644 --- a/modules/webapp/src/main/webjar/docspell.js +++ b/modules/webapp/src/main/webjar/docspell.js @@ -97,3 +97,29 @@ elmApp.ports.initClipboard.subscribe(function(args) { docspell_clipboards[page] = new ClipboardJS(sel); } }); + +elmApp.ports.checkSearchQueryString.subscribe(function(args) { + var qStr = args; + if (qStr && DsItemQueryParser && DsItemQueryParser['parseToFailure']) { + var result = DsItemQueryParser.parseToFailure(qStr); + var answer; + if (result) { + answer = + { success: false, + input: result.input, + index: result.failedAt, + messages: result.messages + }; + + } else { + answer = + { success: true, + input: qStr, + index: 0, + messages: [] + }; + } + console.log("Sending: " + answer.success); + elmApp.ports.receiveCheckQueryResult.send(answer); + } +}); diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2850dcc7..e964d621 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,6 +1,7 @@ package docspell.build import sbt._ +import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._ object Dependencies { @@ -8,6 +9,7 @@ object Dependencies { val BetterMonadicForVersion = "0.3.1" val BitpeaceVersion = "0.6.0" val CalevVersion = "0.4.1" + val CatsParseVersion = "0.3.1" val CirceVersion = "0.13.0" val ClipboardJsVersion = "2.0.6" val DoobieVersion = "0.10.0" @@ -26,11 +28,13 @@ object Dependencies { val LogbackVersion = "1.2.3" val MariaDbVersion = "2.7.2" val MiniTestVersion = "2.9.3" + val MUnitVersion = "0.7.22" val OrganizeImportsVersion = "0.5.0" val PdfboxVersion = "2.0.22" val PoiVersion = "4.1.2" val PostgresVersion = "42.2.19" val PureConfigVersion = "0.14.1" + val ScalaJavaTimeVersion = "2.2.0" val Slf4jVersion = "1.7.30" val StanfordNlpVersion = "4.2.0" val TikaVersion = "1.25" @@ -41,6 +45,20 @@ object Dependencies { val JQueryVersion = "3.5.1" val ViewerJSVersion = "0.5.8" + val catsParse = Seq( + "org.typelevel" %% "cats-parse" % CatsParseVersion + ) + val catsParseJS = + Def.setting("org.typelevel" %%% "cats-parse" % CatsParseVersion) + + val scalaJsStubs = + "org.scala-js" %% "scalajs-stubs" % "1.0.0" % "provided" + + val catsJS = Def.setting("org.typelevel" %%% "cats-core" % "2.4.2") + + val scalaJavaTime = + Def.setting("io.github.cquiroz" %%% "scala-java-time" % ScalaJavaTimeVersion) + val kittens = Seq( "org.typelevel" %% "kittens" % KittensVersion ) @@ -254,6 +272,11 @@ object Dependencies { "io.monix" %% "minitest-laws" % MiniTestVersion ).map(_ % Test) + val munit = Seq( + "org.scalameta" %% "munit" % MUnitVersion, + "org.scalameta" %% "munit-scalacheck" % MUnitVersion + ) + val kindProjectorPlugin = "org.typelevel" %% "kind-projector" % KindProjectorVersion val betterMonadicFor = "com.olegpy" %% "better-monadic-for" % BetterMonadicForVersion diff --git a/project/plugins.sbt b/project/plugins.sbt index 958f4d6d..a3b25d78 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -7,5 +7,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.0") addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") +addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.5.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.5") diff --git a/website/site/content/docs/api/intro.md b/website/site/content/docs/api/intro.md index 6d9a7818..c6359ab0 100644 --- a/website/site/content/docs/api/intro.md +++ b/website/site/content/docs/api/intro.md @@ -116,3 +116,74 @@ $ curl -H 'X-Docspell-Auth: 1568142446077-ZWlrZS9laWtl-$2a$10$3B0teJ9rMpsBJPzHfZ ,"tagCloud":{"items":[]} } ``` + +### Search for items + +``` bash +$ curl -i -H 'X-Docspell-Auth: 1615240493…kYtFynj4' \ + 'http://localhost:7880/api/v1/sec/item/search?q=tag=todo,invoice%20year:2021' +{ + "groups": [ + { + "name": "2021-02", + "items": [ + { + "id": "41J962DjS7T-sjP9idxJ6o9-hJrmBk34YJN-mQqysHwcFD6", + "name": "something.txt", + "state": "confirmed", + "date": 1613598750202, + "dueDate": 1617883200000, + "source": "webapp", + "direction": "outgoing", + "corrOrg": { + "id": "J58tYifCh4X-cze5R8eSJcc-YAFr6qt1VKL-1ZmhRwiTXoH", + "name": "EasyCare AG" + }, + "corrPerson": null, + "concPerson": null, + "concEquipment": null, + "folder": { + "id": "GKwSvYVdvfb-QeAwzzT7pBM-Gbji2hQc2bL-uCyrMCAg3wo", + "name": "test" + }, + "attachments": [], + "tags": [], + "customfields": [], + "notes": null, + "highlighting": [] + } + ] + }, + { + "name": "2021-01", + "items": [ + { + "id": "ANqtuDynXWU-PrhzUxzQVmH-PDuJfeJ6dYB-Ut3g1jrcFhw", + "name": "letter-de.pdf", + "state": "confirmed", + "date": 1611144000000, + "dueDate": null, + "source": "webapp", + "direction": "incoming", + "corrOrg": { + "id": "J58tYifCh4X-cze5R8eSJcc-YAFr6qt1VKL-1ZmhRwiTXoH", + "name": "EasyCare AG" + }, + "corrPerson": null, + "concPerson": { + "id": "AA5sV1nH9ve-mDCn4DxDRvu-tWkUquiW4fZ-fVJimW4Vq79", + "name": "Max Mustermann" + }, + "concEquipment": null, + "folder": null, + "attachments": [], + "tags": [], + "customfields": [], + "notes": null, + "highlighting": [] + } + ] + } + ] +} +``` diff --git a/website/site/content/docs/features/_index.md b/website/site/content/docs/features/_index.md index 2160da95..cf213621 100644 --- a/website/site/content/docs/features/_index.md +++ b/website/site/content/docs/features/_index.md @@ -17,6 +17,8 @@ description = "A list of features and limitations." - Conversion to PDF: all files are converted into a PDF file. PDFs with only images (as often returned from scanners) are converted into searchable PDF/A pdfs. +- A powerful [query language](@/docs/query/_index.md) to find + documents - Non-destructive: all your uploaded files are never modified and can always be downloaded untouched - Organize files using tags, folders, [Custom diff --git a/website/site/content/docs/query/_index.md b/website/site/content/docs/query/_index.md new file mode 100644 index 00000000..dad3d732 --- /dev/null +++ b/website/site/content/docs/query/_index.md @@ -0,0 +1,527 @@ ++++ +title = "Query Language" +weight = 55 +description = "The query language is a powerful way to search for documents." +insert_anchor_links = "right" +[extra] +mktoc = true ++++ + + +Docspell uses a query language to provide a powerful way to search for +your documents. It is targeted at advanced users and it needs to be +enabled explicitely in your user settings. + +
+{{ figure(file="enable-powersearch.png") }} +
+ +This changes the search bar on the items list page to expect a query +as described below. + +The search menu works as before, the query coming from the search menu +is combined with a query from the search bar. + +For taking a quick look, head over to the [examples](#examples). + +# Structure + +The overall query is an expression that evaluates to `true` or `false` +when applied to an item and so selects whether to include it in the +results or not. It consists of smaller expressions that can be +combined via the common ways: `and`, `or` and `not`. + +Simple expressions check some property of an item. The form is: + +``` + +``` + +For example: `tag=invoice` – where `tag` is the field, `=` the +operator and `invoice` the value. It would evaluate to `true` if the +item has a tag with name `invoice` and to `false` if the item doesn't +have a tag with name `invoice`. + +Multiple expressions are separated by whitespace and are combined via +`AND` by default. To explicitely combine them, wrap a list of +expressions into one of these: + +- `(& … )` to combine them via `AND` +- `(| … )` to combine them via `OR` + +It is also possible to negate an expression, by prefixing it with a +`!`; for example `!tag=invoice`. + +# The Parts + +## Operators + +There are 7 operators: + +- `=` for equals +- `>` for greater-than +- `>=` for greater-equals +- `~=` for "in" (a shorter way to say "a or b or c or d") +- `:` for "like" +- `<` for lower than +- `<=` for lower-equal +- `!=` for not-equals + +Not all operators work with every field. + +## Fields + +Fields are used to identify a property of an item. They also define +what operators are allowed. There are fields where an item can have at +most one value (like `name` or `notes`) and there are fields where an +item can have multiple values (like `tag`). At last there are special +fields that are either implemented directly using custom sql or that +are shortcuts to a longer form. + +Here is the list of all available fields. + +These fields map to at most one value: + +- `name` the item name +- `source` the source used for uploading +- `notes` the item notes +- `id` the item id +- `date` the item date +- `due` the due date of the item +- `attach.count` the number of attachments of the item +- `corr.org.id` the id of the correspondent organization +- `corr.org.name` the name of the correspondent organization +- `corr.pers.name` name of correspondent person +- `corr.pers.id` id of correspondent person +- `conc.pers.name` name of concerning person +- `conc.pers.id` id of concerning person +- `conc.equip.name` name of equipment +- `conc.equip.id` id of equipment +- `folder.id` id of a folder +- `folder` name of a folder +- `inbox` whether to return "new" items (boolean) +- `incoming` whether to return incoming items (boolean), `true` to + show only incoming, `false` to show only outgoing. + +These fields support all operators, except `incoming` and `inbox` +which expect boolean values and there these operators don't make much +sense. + +Fields that map to more than one value: + +- `tag` the tag name +- `tag.id` the tag id +- `cat` name of the tag category + +The tag and category fields use two operators: `:` and `=`. + +Other special fields: + +- `attach.id` +- `checksum` +- `content` +- `f` for referencing custom fields by name +- `f.id` for referencing custom fields by their id +- `dateIn` a shortcut for a range search +- `dueIn` a shortcut for a range search +- `exist` check if some porperty exists +- `names` +- `year` +- `conc` +- `corr` + +These fields are often using the `:` operator to simply separate field +and value. They are often backed by a custom implementation, or they +are shortcuts for a longer query. + +## Values + +Values are the data you want to search for. There are different kinds +of that, too: there are text-based values, numbers, boolean and dates. +When multiple values are allowed, they must be separated by comma `,`. + +### Text Values + +Text values need to be put in quotes (`"`) if they contain one of +these characters: +- whitespace ` ` +- quotes `"` +- backslash `\` +- comma `,` +- brackets `[]` +- parens `()` + +Any quotes inside a quoted string must be escaped with a backslash. +Examples: `scan_123`, `a-b-c`, `x.y.z`, `"scan from today"`, `"a \"strange\" +name.pdf"` + +### Numeric and Boolean Values + +Numeric values can be entered literally; an optional fraction part is +separetd by a dot. Examples: `1`, `2.15`. + +A boolean value can be specfied by `yes` or `true` and `no` or +`false`, respectively. Example: `inbox:yes` + +### Dates + +Dates are always treated as local dates and can be entered in multiple +ways. + +#### Date Pattern + +They can be in the following form: `YYYY-MM-DD` or `YYYY/MM/DD`. +The month and day part are optional; if they are missing they are +filled automatically with a `1`. So `2020-01` would be the same as +`2020-01-01`. + +A special pattern is `today` which marks the current day. + +#### Unix Epoch + +Dates can be given in milliseconds from unix epoch. Then it must be +prefixed by `ms`. The time part is ignored. Examples: +`ms1615209591627`. + +#### Calculation + +Dates can be defined by providing a base date and a period to add or +substract. This is especially useful with the `today` pattern. The +period must be separated from the date by a semi-colon `;`. Then write +a `+` or a `-` to add or substract and at last the number of days +(suffix `d`) or months (suffix `m`). + +Examples: `today;-14d`, `2020-02;+1m` + +# Simple Expressions + +Simple expressions are made up of a field with at most one value, an +operator and one or more values. These fields support all operators, +except for boolean fields. + +The like operator `:` can be used with all values, but makes only +sense for text values. It allows to do a substring search for a field. +For example, to look for an item with a name of exactly 'invoice_22': + +``` +name=invoice_22 +``` + +Using `:` it is possible to look for items that have 'invoice' in +their name: + +``` +name:*invoice* +``` + +The asterisk `*` can be added at the beginning and/or end of the +value. Furthermore, the like operator is case-insensitive, whereas `=` +is not. This applies to all fields with a text value; this is another +example looking for a correspondent person of with 'marcus' in the +name: +``` +corr.pers.name:*marcus* +``` + + +---- + +Comparisons via `<`/`>` are done alphanumerically for text based +values and numerically for numeric values. For booleans these +operators don't make sense and therefore don't work there. + +---- + +All these fields (except boolean fields) allow to use the in-operator, +`~=`. This is a more efficient form to specify a list of alternatives +and is logically the same as combining multiple expressions with +`OR`. For example: + +``` +source~=webapp,mailbox +``` + +is the same as +``` +(| source=webapp source=mailbox ) +``` + +The `~=` version is nicer to read, safes some key strokes and also +runs more efficient when the list grows. It is *not* possible to use a +wildcard `*` here. If a wildcard is required, you need to write the +longer form. + +If one value contains whitespace or other characters that require +quoting, each value must be quoted, not the whole list. So this is +correct: +``` +source~="web app","mail box" +``` + +This is not correct: `source~="web app,mail box"` – it would be treated +as one single value and is then essentially the same as using `=`. + +---- + +The two fields `incoming` and `inbox` expect a boolean value: one of +`true` or `false`. The synonyms `yes` and `no` can also be used to +make it better readable. + +This finds all items that have not been confirmed: +``` +inbox:yes +``` + +The `incoming` can be used to show only incoming or only outgoing +documents: + +``` +incoming:yes +``` + +For outgoing, you need to say: +``` +incoming:no +``` + + +# Tags + +Tags have their own syntax, because they are an important tool for +organizing items. Tags only allow for two operators: `=` and `:`. +Combined with negation (the `!` operator), this is quite flexible. + +For tags, `=` means that items must have *all* specified tags (or +more), while `:` means that items must have at least *one* of the +specified tags. Tags can be identified by their name or id and are +given as a comma separated list (just like when using the +in-operator). + +Some examples: Find all invoices that are todo: +``` +tag=invoice,todo +``` + +This returns all items that have tags `invoice` and `todo` – and +possible some other tags. Negating this: +``` +!tag=invoice,todo +``` + +… results in an expression that returns all items that don't have +*both* tags. It might return items with tag `invoice` and also items +with tag `todo`, but no items that have both of them. + +Using `:` is just analog to `=`. This finds all items that are either +`waiting` or `todo` (or both): + +``` +tag:waiting,todo +``` + +When negating this: +``` +!tag:waiting,todo +``` + +it finds all items that have *none* of the tags. + +Tag names are always compared case-insensitive. Tags can also be +selected using their id, then the field name `tag.id` must be used +instead of `tag`. + +The field `cat` can be used the same way to search for tag categories. + +# Custom Fields + +Custom fields are implemented via the following syntax: + +``` +f: +``` + +They look almost like a simple expression, only prefixed with a `f:` +to indicate that the following is the name of a custom field. + +The type of a custom field is honored. So if you have a money or +numeric type, comparsions are done numerically. Otherwise a +alphnumeric comparison is performed. Custom fields do not support the +in-operator (`~=`). + +For example: assuming there is a custom field of type *money* and name +*usd*, the following selects all items with an amount between 10 and +150: + +``` +f:usd>10 f:usd<150 +``` + +The like-operator can be used, too. For example, to find all items +that have a custom field `asn` (often used for a serial number printed +on the document): + +``` +f:asn:* +``` + +If the like operator is used on numeric fields, it falls back to +text-comparison. + +Instead of using the name, the field-id can be used to select a field. +Then the prefix is `f.id`: + +``` +f.id:J2ES1Z4Ni9W-xw1VdFbt3KA-rL725kuyVzh-7La95Yw7Ax2:15.00 +``` + + +# Fulltext Search + +The special field `content` allows to add a fulltext search. Using +this is currently restricted: it must occur in the root query and +cannot be nested in other complex expressions. + +The form is: + +``` +content: +``` + +The search query is interpreted by the fulltext index (currently it is +SOLR). This is usually very powerful and in many cases this value must +be quoted. + +For example, do a fulltext search for 'red needle': +``` +content:"red needle" +``` + +It can be combined in an AND expression (but not deeper): +``` +content:"red needle" tag:todo +``` + + +# File Checksums + +The `checksum` field can be used to look for items that have a certain +file attached. It expects a SHA256 string. + +For example, this is the sha256 checksum of some file on the hard +disk: +`40675c22ab035b8a4ffe760732b65e5f1d452c59b44d3d0a2a08a95d28853497`. + +To find all items that have (exactly) this file attached: +``` +checksum:40675c22ab035b8a4ffe760732b65e5f1d452c59b44d3d0a2a08a95d28853497 +``` + +# Exist + +The `exist` field can be used with another field, to check whether an +item has some value for a given field. It only works for fields that +have at most one value. + +For example, it could be used to find fields that are in any folder: + +``` +exist:folder +``` + +When negating, it finds all items that are not in a folder: +``` +!exist:folder +``` + + +# Attach-Id + +The `attach.id` field is a special field to find items by providing +the id of an attachment. This can be helpful in certain situations +when you only have the id of an attachment. It always uses equality, +so all other operators are not supported. + +``` +attach.id=5YjdnuTAdKJ-V6ofWTYsqKV-mAwB5aXTNWE-FAbeRU58qLb +``` + +# Shortcuts + +Shortcuts are only a short form of a longer query and are provided for +convenience. The following exist: + +- `dateIn` and `dueIn` +- `year` +- `names` +- `conc` +- `corr` + + +### Date Ranges + +The first three are all short forms to specify a range search. With +`dateIn` and `dueIn` have three forms that are translated into a range +search: + +- `dateIn:2020-01;+15d` → `date>=2020-01 date<2020-01;+15d` +- `dateIn:2020-01;-15d` → `date>=2020-01;-15d date<2020-01` +- `dateIn:2020-01;/15d` → `date>=2020-01;-15d date<2020-01;+15d` + +The syntax is the same as defining a date by adding a period to some +base date. These two dates are used to expand the form into a range +search. There is an additional `/` character to allow to subtract and +add the period. + +The `year` is almost the same thing, only a lot shorter to write. It +expands into a range search (only for the item date!) that selects all +items with a date in the specified year: + +- `year:2020` → `date>=2020-01-01 date<2021-01-01` + +The last shortcut is `names`. It allows to search in many "names" of +related entities at once: + +### Names + +- `names:tim` → `(| name:tim corr.org.name:tim corr.pers.name:tim conc.pers.name:tim conc.equip.name:tim )` + +The `names` field uses the like-operator. + +The fields `conc` and `corr` are analog to `names`, only that they +look into correspondent names and concerning names. + +- `conc:marc*` → `(| conc.pers.name:marc* conc.equip.name:marc* )` +- `corr:marc*` → `(| corr.org.name:marc* corr.pers.name:marc* )` + + +# Examples + +Find items with 2 or more attachments: +``` +attach.count>2 +``` + +Find items with at least one tag invoice or todo: +``` +tag:invoice,todo +``` + +Find items with at least both tags invoice and todo: +``` +tag=invoice,todo +``` + +Find items with a concerning person of name starting with "Marcus": +``` +conc.pers.name:marcus* +``` + +Find items with at least a tag "todo" in year 2020: +``` +tag:todo year:2020 +``` + +Find items within the last 30 days: +``` +date>today;-30d +``` diff --git a/website/site/content/docs/query/enable-powersearch.png b/website/site/content/docs/query/enable-powersearch.png new file mode 100644 index 00000000..ec5dbcc4 Binary files /dev/null and b/website/site/content/docs/query/enable-powersearch.png differ