From 5882405f30b7661f7fdb06571f62213a43089112 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sat, 5 Dec 2020 02:59:57 +0100 Subject: [PATCH] Search index if search object only contains this field --- build.sbt | 8 ++- .../src/main/resources/docspell-openapi.yml | 3 +- .../restserver/routes/ItemRoutes.scala | 70 +++++++++++++++++-- project/Dependencies.scala | 6 ++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 9088695e..02f2401b 100644 --- a/build.sbt +++ b/build.sbt @@ -184,7 +184,10 @@ val openapiScalaSettings = Seq( case "glob" => field => field.copy(typeDef = TypeDef("Glob", Imports("docspell.common.Glob"))) case "customfieldtype" => - field => field.copy(typeDef = TypeDef("CustomFieldType", Imports("docspell.common.CustomFieldType"))) + field => + field.copy(typeDef = + TypeDef("CustomFieldType", Imports("docspell.common.CustomFieldType")) + ) })) ) @@ -465,6 +468,7 @@ val restserver = project Dependencies.circe ++ Dependencies.pureconfig ++ Dependencies.yamusca ++ + Dependencies.kittens ++ Dependencies.webjars ++ Dependencies.loggingApi ++ Dependencies.logging.map(_ % Runtime), @@ -681,7 +685,7 @@ def packageTools(logger: Logger, dir: File, version: String): Seq[File] = { (dir ** "*") .filter(f => !excludes.exists(p => f.absolutePath.startsWith(p.absolutePath))) .pair(sbt.io.Path.relativeTo(dir)) - .map({case (f, name) => (f, s"docspell-tools-${version}/$name") }) + .map({ case (f, name) => (f, s"docspell-tools-${version}/$name") }) IO.zip( Seq( diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 99d6d884..a51ff052 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -5033,7 +5033,8 @@ components: fullText: type: string description: | - A query searching the contents of documents. + A query searching the contents of documents. If only this + field is set, then a fulltext-only search is done. corrOrg: type: string format: ident diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index c6606667..1a088a39 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -3,6 +3,7 @@ package docspell.restserver.routes import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ +import cats.Monoid import docspell.backend.BackendApp import docspell.backend.auth.AuthToken @@ -10,7 +11,7 @@ import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OFulltext import docspell.backend.ops.OItemSearch.Batch import docspell.common.syntax.all._ -import docspell.common.{Ident, ItemState} +import docspell.common._ import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -51,8 +52,19 @@ object ItemRoutes { _ <- logger.ftrace(s"Got search mask: $mask") query = Conversions.mkQuery(mask, user.account) _ <- logger.ftrace(s"Running query: $query") - resp <- mask.fullText match { - case Some(fq) if cfg.fullTextSearch.enabled => + resp <- mask match { + case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => + val ftsIn = OFulltext.FtsInput(ftq.query) + for { + items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( + ftsIn, + user.account, + Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) + ) + ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) + } yield ok + + case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => for { items <- backend.fulltext.findItems(cfg.maxNoteLength)( query, @@ -61,6 +73,7 @@ object ItemRoutes { ) ok <- Ok(Conversions.mkItemListFts(items)) } yield ok + case _ => for { items <- backend.itemSearch.findItems(cfg.maxNoteLength)( @@ -78,8 +91,19 @@ object ItemRoutes { _ <- logger.ftrace(s"Got search mask: $mask") query = Conversions.mkQuery(mask, user.account) _ <- logger.ftrace(s"Running query: $query") - resp <- mask.fullText match { - case Some(fq) if cfg.fullTextSearch.enabled => + resp <- mask match { + case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => + val ftsIn = OFulltext.FtsInput(ftq.query) + for { + items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( + ftsIn, + user.account, + Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) + ) + ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) + } yield ok + + case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => for { items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)( query, @@ -390,4 +414,40 @@ object ItemRoutes { def notEmpty: Option[String] = opt.map(_.trim).filter(_.nonEmpty) } + + object SearchFulltextOnly { + implicit private val identMonoid: Monoid[Ident] = + Monoid.instance(Ident.unsafe(""), _ / _) + + implicit private val timestampMonoid: Monoid[Timestamp] = + Monoid.instance(Timestamp.Epoch, (a, _) => a) + + implicit private val directionMonoid: Monoid[Direction] = + Monoid.instance(Direction.Incoming, (a, _) => a) + + implicit private val idListMonoid: Monoid[IdList] = + Monoid.instance(IdList(Nil), (a, b) => IdList(a.ids ++ b.ids)) + + implicit private val boolMonoid: Monoid[Boolean] = + Monoid.instance(false, _ || _) + + private val itemSearchMonoid: Monoid[ItemSearch] = + cats.derived.semiauto.monoid + + def unapply(m: ItemSearch): Option[ItemFtsSearch] = + m.fullText match { + case Some(fq) => + val me = m.copy(fullText = None, offset = 0, limit = 0) + if (itemSearchMonoid.empty == me) + Some(ItemFtsSearch(m.offset, m.limit, fq)) + else None + case _ => + None + } + } + + object SearchWithFulltext { + def unapply(m: ItemSearch): Option[String] = + m.fullText + } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 62629234..a6753b0c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -21,6 +21,7 @@ object Dependencies { val Icu4jVersion = "68.1" val JsoupVersion = "1.13.1" val KindProjectorVersion = "0.10.3" + val KittensVersion = "2.2.0" val LevigoJbig2Version = "2.0" val Log4sVersion = "1.9.0" val LogbackVersion = "1.2.3" @@ -41,6 +42,11 @@ object Dependencies { val JQueryVersion = "3.5.1" val ViewerJSVersion = "0.5.8" + + val kittens = Seq( + "org.typelevel" %% "kittens" % KittensVersion + ) + val calevCore = Seq( "com.github.eikek" %% "calev-core" % CalevVersion )