diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala index 8eb91c7f..e468eaaf 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala @@ -5,7 +5,7 @@ import cats.implicits._ import docspell.backend.ops.OSimpleSearch._ import docspell.common._ -import docspell.query.{ItemQueryParser, ParseFailure} +import docspell.query._ import docspell.store.qb.Batch import docspell.store.queries.Query import docspell.store.queries.SearchSummary @@ -20,15 +20,29 @@ trait OSimpleSearch[F[_]] { def searchByString( settings: Settings - )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]] + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] def searchSummaryByString( useFTS: Boolean - )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] } object OSimpleSearch { + sealed trait StringSearchResult[+A] + object StringSearchResult { + case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing] + def parseFailed[A](error: ParseFailure): StringSearchResult[A] = + ParseFailed(error) + + case class FulltextMismatch(error: FulltextExtract.FailureResult) + extends StringSearchResult[Nothing] + def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] = + FulltextMismatch(error) + + case class Success[A](value: A) extends StringSearchResult[A] + } + final case class Settings( batch: Batch, useFTS: Boolean, @@ -104,19 +118,49 @@ object OSimpleSearch { extends OSimpleSearch[F] { def searchByString( settings: Settings - )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]] = - ItemQueryParser - .parse(q.query) - .map(iq => Query(fix, Query.QueryExpr(iq))) - .map(search(settings)(_, None)) //TODO resolve content:xyz expressions + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] = { + val parsed: Either[StringSearchResult[Items], ItemQuery] = + ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed) + + def makeQuery(iq: ItemQuery): F[StringSearchResult[Items]] = + iq.findFulltext match { + case FulltextExtract.Result.Success(expr, ftq) => + search(settings)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + .map(StringSearchResult.Success.apply) + case other: FulltextExtract.FailureResult => + StringSearchResult.fulltextMismatch[Items](other).pure[F] + } + + parsed match { + case Right(iq) => + makeQuery(iq) + case Left(err) => + err.pure[F] + } + } def searchSummaryByString( useFTS: Boolean - )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] = - ItemQueryParser - .parse(q.query) - .map(iq => Query(fix, Query.QueryExpr(iq))) - .map(searchSummary(useFTS)(_, None)) //TODO resolve content:xyz expressions + )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] = { + val parsed: Either[StringSearchResult[SearchSummary], ItemQuery] = + ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed) + + def makeQuery(iq: ItemQuery): F[StringSearchResult[SearchSummary]] = + iq.findFulltext match { + case FulltextExtract.Result.Success(expr, ftq) => + searchSummary(useFTS)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + .map(StringSearchResult.Success.apply) + case other: FulltextExtract.FailureResult => + StringSearchResult.fulltextMismatch[SearchSummary](other).pure[F] + } + + parsed match { + case Right(iq) => + makeQuery(iq) + case Left(err) => + err.pure[F] + } + } def searchSummary( useFTS: Boolean diff --git a/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala b/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala new file mode 100644 index 00000000..769df244 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala @@ -0,0 +1,71 @@ +package docspell.query + +import cats._ +import cats.implicits._ + +import docspell.query.ItemQuery.Expr.AndExpr +import docspell.query.ItemQuery.Expr.NotExpr +import docspell.query.ItemQuery.Expr.OrExpr +import docspell.query.ItemQuery._ + +/** Currently, fulltext in a query is only supported when in "root + * AND" position + */ +object FulltextExtract { + + sealed trait Result + sealed trait SuccessResult extends Result + sealed trait FailureResult extends Result + object Result { + case class Success(query: Expr, fts: Option[String]) extends SuccessResult + case object TooMany extends FailureResult + case object UnsupportedPosition extends FailureResult + } + + def findFulltext(expr: Expr): Result = + lookForFulltext(expr) + + private def lookForFulltext(expr: Expr): Result = + expr match { + case Expr.Fulltext(ftq) => + Result.Success(ItemQuery.all.expr, ftq.some) + case Expr.AndExpr(inner) => + inner.collect({ case Expr.Fulltext(fq) => fq }) match { + case Nil => + checkPosition(expr, 0) + case e :: Nil => + val c = foldMap(isFulltextExpr)(expr) + if (c > 1) Result.TooMany + else Result.Success(expr, e.some) + case _ => + Result.TooMany + } + case _ => + checkPosition(expr, 0) + } + + private def checkPosition(expr: Expr, max: Int): Result = { + val c = foldMap(isFulltextExpr)(expr) + if (c > max) Result.UnsupportedPosition + else Result.Success(expr, None) + } + + private def foldMap[B: Monoid](f: Expr => B)(expr: Expr): B = + expr match { + case OrExpr(inner) => + inner.map(foldMap(f)).fold + case AndExpr(inner) => + inner.map(foldMap(f)).fold + case NotExpr(e) => + f(e) + case _ => + f(expr) + } + + private def isFulltextExpr(expr: Expr): Int = + expr match { + case Expr.Fulltext(_) => 1 + case _ => 0 + } + +} diff --git a/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala index acf2307a..eb927a87 100644 --- a/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala +++ b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala @@ -11,7 +11,10 @@ import docspell.query.ItemQuery.Attr.{DateAttr, StringAttr} * against a specific field of an item using some operator or a * combination thereof. */ -final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) +final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) { + def findFulltext: FulltextExtract.Result = + FulltextExtract.findFulltext(expr) +} object ItemQuery { val all = ItemQuery(Expr.Exists(Attr.ItemId), Some("")) diff --git a/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala b/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala new file mode 100644 index 00000000..0c3b555a --- /dev/null +++ b/modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala @@ -0,0 +1,57 @@ +package docspell.query + +import cats.implicits._ +import munit._ +import docspell.query.FulltextExtract.Result + +class FulltextExtractTest extends FunSuite { + + def findFts(q: String): Result = { + val p = ItemQueryParser.parseUnsafe(q) + FulltextExtract.findFulltext(p.expr) + } + + def assertFts(qstr: String, expect: Result) = + assertEquals(findFts(qstr), expect) + + def assertFtsSuccess(qstr: String, expect: Option[String]) = { + val q = ItemQueryParser.parseUnsafe(qstr) + assertEquals(findFts(qstr), Result.Success(q.expr, expect)) + } + + test("find fulltext as root") { + assertEquals(findFts("content:what"), Result.Success(ItemQuery.all.expr, "what".some)) + assertEquals( + findFts("content:\"what hello\""), + Result.Success(ItemQuery.all.expr, "what hello".some) + ) + assertEquals( + findFts("content:\"what OR hello\""), + Result.Success(ItemQuery.all.expr, "what OR hello".some) + ) + } + + test("find no fulltext") { + assertFtsSuccess("name:test", None) + } + + test("find fulltext within and") { + assertFtsSuccess("content:what name:test", "what".some) + assertFtsSuccess("$names:marc* content:what name:test", "what".some) + assertFtsSuccess( + "$names:marc* date:2021-02 content:\"what else\" name:test", + "what else".some + ) + } + + test("too many fulltext searches") { + assertFts("content:yes content:no", Result.TooMany) + assertFts("content:yes (| name:test content:no)", Result.TooMany) + assertFts("content:yes (| name:test (& date:2021-02 content:no))", Result.TooMany) + } + + test("wrong fulltext search position") { + assertFts("name:test (| date:2021-02 content:yes)", Result.UnsupportedPosition) + assertFts("name:test (& date:2021-02 content:yes)", Result.UnsupportedPosition) //TODO + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index 9a0fd9cc..9422fdb0 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -11,8 +11,11 @@ import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OFulltext import docspell.backend.ops.OItemSearch.{Batch, Query} import docspell.backend.ops.OSimpleSearch +import docspell.backend.ops.OSimpleSearch.StringSearchResult import docspell.common._ import docspell.common.syntax.all._ +import docspell.query.FulltextExtract.Result.TooMany +import docspell.query.FulltextExtract.Result.UnsupportedPosition import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -60,31 +63,12 @@ object ItemRoutes { cfg.maxNoteLength ) val fixQuery = Query.Fix(user.account, None, None) - backend.simpleSearch.searchByString(settings)(fixQuery, itemQuery) match { - case Right(results) => - val items = results.map( - _.fold( - Conversions.mkItemListFts, - Conversions.mkItemListWithTagsFts, - Conversions.mkItemList, - Conversions.mkItemListWithTags - ) - ) - Ok(items) - case Left(fail) => - BadRequest(BasicResult(false, fail.render)) - } + searchItems(backend, dsl)(settings, fixQuery, itemQuery) case GET -> Root / "searchStats" :? QP.Query(q) => val itemQuery = ItemQueryString(q) val fixQuery = Query.Fix(user.account, None, None) - backend.simpleSearch - .searchSummaryByString(cfg.fullTextSearch.enabled)(fixQuery, itemQuery) match { - case Right(summary) => - summary.flatMap(s => Ok(Conversions.mkSearchStats(s))) - case Left(fail) => - BadRequest(BasicResult(false, fail.render)) - } + searchItemStats(backend, dsl)(cfg.fullTextSearch.enabled, fixQuery, itemQuery) case req @ POST -> Root / "search" => for { @@ -103,21 +87,7 @@ object ItemRoutes { cfg.maxNoteLength ) fixQuery = Query.Fix(user.account, None, None) - resp <- backend.simpleSearch - .searchByString(settings)(fixQuery, itemQuery) match { - case Right(results) => - val items = results.map( - _.fold( - Conversions.mkItemListFts, - Conversions.mkItemListWithTagsFts, - Conversions.mkItemList, - Conversions.mkItemListWithTags - ) - ) - Ok(items) - case Left(fail) => - BadRequest(BasicResult(false, fail.render)) - } + resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery) } yield resp case req @ POST -> Root / "searchStats" => @@ -125,16 +95,11 @@ object ItemRoutes { userQuery <- req.as[ItemQuery] itemQuery = ItemQueryString(userQuery.query) fixQuery = Query.Fix(user.account, None, None) - resp <- backend.simpleSearch - .searchSummaryByString(cfg.fullTextSearch.enabled)( - fixQuery, - itemQuery - ) match { - case Right(summary) => - summary.flatMap(s => Ok(Conversions.mkSearchStats(s))) - case Left(fail) => - BadRequest(BasicResult(false, fail.render)) - } + resp <- searchItemStats(backend, dsl)( + cfg.fullTextSearch.enabled, + fixQuery, + itemQuery + ) } yield resp //DEPRECATED @@ -526,6 +491,63 @@ object ItemRoutes { } } + def searchItems[F[_]: Sync]( + backend: BackendApp[F], + dsl: Http4sDsl[F] + )(settings: OSimpleSearch.Settings, fixQuery: Query.Fix, itemQuery: ItemQueryString) = { + import dsl._ + + backend.simpleSearch + .searchByString(settings)(fixQuery, itemQuery) + .flatMap { + case StringSearchResult.Success(items) => + Ok( + items.fold( + Conversions.mkItemListFts, + Conversions.mkItemListWithTagsFts, + Conversions.mkItemList, + Conversions.mkItemListWithTags + ) + ) + case StringSearchResult.FulltextMismatch(TooMany) => + BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) + case StringSearchResult.FulltextMismatch(UnsupportedPosition) => + BadRequest( + BasicResult( + false, + "Fulltext search must be in root position or inside the first AND." + ) + ) + case StringSearchResult.ParseFailed(pf) => + BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) + } + } + + def searchItemStats[F[_]: Sync]( + backend: BackendApp[F], + dsl: Http4sDsl[F] + )(ftsEnabled: Boolean, fixQuery: Query.Fix, itemQuery: ItemQueryString) = { + import dsl._ + backend.simpleSearch + .searchSummaryByString(ftsEnabled)(fixQuery, itemQuery) + .flatMap { + case StringSearchResult.Success(summary) => + Ok(Conversions.mkSearchStats(summary)) + case StringSearchResult.FulltextMismatch(TooMany) => + BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) + case StringSearchResult.FulltextMismatch(UnsupportedPosition) => + BadRequest( + BasicResult( + false, + "Fulltext search must be in root position or inside the first AND." + ) + ) + case StringSearchResult.ParseFailed(pf) => + BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) + } + + } + implicit final class OptionString(opt: Option[String]) { def notEmpty: Option[String] = opt.map(_.trim).filter(_.nonEmpty)