Resolve fulltext search queries the same way as before

For now, fulltext search is only possible when being the only term or
inside the root AND expression.

modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala

@@ -5,7 +5,7 @@ import cats.implicits._
 
 import docspell.backend.ops.OSimpleSearch._
 import docspell.common._
-import docspell.query.{ItemQueryParser, ParseFailure}
+import docspell.query._
 import docspell.store.qb.Batch
 import docspell.store.queries.Query
 import docspell.store.queries.SearchSummary
@@ -20,15 +20,29 @@ trait OSimpleSearch[F[_]] {
 
   def searchByString(
       settings: Settings
-  )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]]
+  )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]]
   def searchSummaryByString(
       useFTS: Boolean
-  )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]]
+  )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]]
 
 }
 
 object OSimpleSearch {
 
+  sealed trait StringSearchResult[+A]
+  object StringSearchResult {
+    case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing]
+    def parseFailed[A](error: ParseFailure): StringSearchResult[A] =
+      ParseFailed(error)
+
+    case class FulltextMismatch(error: FulltextExtract.FailureResult)
+        extends StringSearchResult[Nothing]
+    def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] =
+      FulltextMismatch(error)
+
+    case class Success[A](value: A) extends StringSearchResult[A]
+  }
+
   final case class Settings(
       batch: Batch,
       useFTS: Boolean,
@@ -104,19 +118,49 @@ object OSimpleSearch {
       extends OSimpleSearch[F] {
     def searchByString(
         settings: Settings
-    )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]] =
+    )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] = {
-      ItemQueryParser
+      val parsed: Either[StringSearchResult[Items], ItemQuery] =
-        .parse(q.query)
+        ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed)
-        .map(iq => Query(fix, Query.QueryExpr(iq)))
+
-        .map(search(settings)(_, None)) //TODO resolve content:xyz expressions
+      def makeQuery(iq: ItemQuery): F[StringSearchResult[Items]] =
+        iq.findFulltext match {
+          case FulltextExtract.Result.Success(expr, ftq) =>
+            search(settings)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq)
+              .map(StringSearchResult.Success.apply)
+          case other: FulltextExtract.FailureResult =>
+            StringSearchResult.fulltextMismatch[Items](other).pure[F]
+        }
+
+      parsed match {
+        case Right(iq) =>
+          makeQuery(iq)
+        case Left(err) =>
+          err.pure[F]
+      }
+    }
 
     def searchSummaryByString(
         useFTS: Boolean
-    )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] =
+    )(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] = {
-      ItemQueryParser
+      val parsed: Either[StringSearchResult[SearchSummary], ItemQuery] =
-        .parse(q.query)
+        ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed)
-        .map(iq => Query(fix, Query.QueryExpr(iq)))
+
-        .map(searchSummary(useFTS)(_, None)) //TODO resolve content:xyz expressions
+      def makeQuery(iq: ItemQuery): F[StringSearchResult[SearchSummary]] =
+        iq.findFulltext match {
+          case FulltextExtract.Result.Success(expr, ftq) =>
+            searchSummary(useFTS)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq)
+              .map(StringSearchResult.Success.apply)
+          case other: FulltextExtract.FailureResult =>
+            StringSearchResult.fulltextMismatch[SearchSummary](other).pure[F]
+        }
+
+      parsed match {
+        case Right(iq) =>
+          makeQuery(iq)
+        case Left(err) =>
+          err.pure[F]
+      }
+    }
 
     def searchSummary(
         useFTS: Boolean

modules/query/shared/src/main/scala/docspell/query/FulltextExtract.scala

@@ -0,0 +1,71 @@
+package docspell.query
+
+import cats._
+import cats.implicits._
+
+import docspell.query.ItemQuery.Expr.AndExpr
+import docspell.query.ItemQuery.Expr.NotExpr
+import docspell.query.ItemQuery.Expr.OrExpr
+import docspell.query.ItemQuery._
+
+/** Currently, fulltext in a query is only supported when in "root
+  * AND" position
+  */
+object FulltextExtract {
+
+  sealed trait Result
+  sealed trait SuccessResult extends Result
+  sealed trait FailureResult extends Result
+  object Result {
+    case class Success(query: Expr, fts: Option[String]) extends SuccessResult
+    case object TooMany                                  extends FailureResult
+    case object UnsupportedPosition                      extends FailureResult
+  }
+
+  def findFulltext(expr: Expr): Result =
+    lookForFulltext(expr)
+
+  private def lookForFulltext(expr: Expr): Result =
+    expr match {
+      case Expr.Fulltext(ftq) =>
+        Result.Success(ItemQuery.all.expr, ftq.some)
+      case Expr.AndExpr(inner) =>
+        inner.collect({ case Expr.Fulltext(fq) => fq }) match {
+          case Nil =>
+            checkPosition(expr, 0)
+          case e :: Nil =>
+            val c = foldMap(isFulltextExpr)(expr)
+            if (c > 1) Result.TooMany
+            else Result.Success(expr, e.some)
+          case _ =>
+            Result.TooMany
+        }
+      case _ =>
+        checkPosition(expr, 0)
+    }
+
+  private def checkPosition(expr: Expr, max: Int): Result = {
+    val c = foldMap(isFulltextExpr)(expr)
+    if (c > max) Result.UnsupportedPosition
+    else Result.Success(expr, None)
+  }
+
+  private def foldMap[B: Monoid](f: Expr => B)(expr: Expr): B =
+    expr match {
+      case OrExpr(inner) =>
+        inner.map(foldMap(f)).fold
+      case AndExpr(inner) =>
+        inner.map(foldMap(f)).fold
+      case NotExpr(e) =>
+        f(e)
+      case _ =>
+        f(expr)
+    }
+
+  private def isFulltextExpr(expr: Expr): Int =
+    expr match {
+      case Expr.Fulltext(_) => 1
+      case _                => 0
+    }
+
+}

modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala

@@ -11,7 +11,10 @@ import docspell.query.ItemQuery.Attr.{DateAttr, StringAttr}
   * against a specific field of an item using some operator or a
   * combination thereof.
   */
-final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String])
+final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) {
+  def findFulltext: FulltextExtract.Result =
+    FulltextExtract.findFulltext(expr)
+}
 
 object ItemQuery {
   val all = ItemQuery(Expr.Exists(Attr.ItemId), Some(""))

modules/query/shared/src/test/scala/docspell/query/FulltextExtractTest.scala

@@ -0,0 +1,57 @@
+package docspell.query
+
+import cats.implicits._
+import munit._
+import docspell.query.FulltextExtract.Result
+
+class FulltextExtractTest extends FunSuite {
+
+  def findFts(q: String): Result = {
+    val p = ItemQueryParser.parseUnsafe(q)
+    FulltextExtract.findFulltext(p.expr)
+  }
+
+  def assertFts(qstr: String, expect: Result) =
+    assertEquals(findFts(qstr), expect)
+
+  def assertFtsSuccess(qstr: String, expect: Option[String]) = {
+    val q = ItemQueryParser.parseUnsafe(qstr)
+    assertEquals(findFts(qstr), Result.Success(q.expr, expect))
+  }
+
+  test("find fulltext as root") {
+    assertEquals(findFts("content:what"), Result.Success(ItemQuery.all.expr, "what".some))
+    assertEquals(
+      findFts("content:\"what hello\""),
+      Result.Success(ItemQuery.all.expr, "what hello".some)
+    )
+    assertEquals(
+      findFts("content:\"what OR hello\""),
+      Result.Success(ItemQuery.all.expr, "what OR hello".some)
+    )
+  }
+
+  test("find no fulltext") {
+    assertFtsSuccess("name:test", None)
+  }
+
+  test("find fulltext within and") {
+    assertFtsSuccess("content:what name:test", "what".some)
+    assertFtsSuccess("$names:marc* content:what name:test", "what".some)
+    assertFtsSuccess(
+      "$names:marc* date:2021-02 content:\"what else\" name:test",
+      "what else".some
+    )
+  }
+
+  test("too many fulltext searches") {
+    assertFts("content:yes content:no", Result.TooMany)
+    assertFts("content:yes (| name:test content:no)", Result.TooMany)
+    assertFts("content:yes (| name:test (& date:2021-02 content:no))", Result.TooMany)
+  }
+
+  test("wrong fulltext search position") {
+    assertFts("name:test (| date:2021-02 content:yes)", Result.UnsupportedPosition)
+    assertFts("name:test (& date:2021-02 content:yes)", Result.UnsupportedPosition) //TODO
+  }
+}

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -11,8 +11,11 @@ import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue}
 import docspell.backend.ops.OFulltext
 import docspell.backend.ops.OItemSearch.{Batch, Query}
 import docspell.backend.ops.OSimpleSearch
+import docspell.backend.ops.OSimpleSearch.StringSearchResult
 import docspell.common._
 import docspell.common.syntax.all._
+import docspell.query.FulltextExtract.Result.TooMany
+import docspell.query.FulltextExtract.Result.UnsupportedPosition
 import docspell.restapi.model._
 import docspell.restserver.Config
 import docspell.restserver.conv.Conversions
@@ -60,31 +63,12 @@ object ItemRoutes {
           cfg.maxNoteLength
         )
         val fixQuery = Query.Fix(user.account, None, None)
-        backend.simpleSearch.searchByString(settings)(fixQuery, itemQuery) match {
+        searchItems(backend, dsl)(settings, fixQuery, itemQuery)
-          case Right(results) =>
-            val items = results.map(
-              _.fold(
-                Conversions.mkItemListFts,
-                Conversions.mkItemListWithTagsFts,
-                Conversions.mkItemList,
-                Conversions.mkItemListWithTags
-              )
-            )
-            Ok(items)
-          case Left(fail) =>
-            BadRequest(BasicResult(false, fail.render))
-        }
 
       case GET -> Root / "searchStats" :? QP.Query(q) =>
         val itemQuery = ItemQueryString(q)
         val fixQuery  = Query.Fix(user.account, None, None)
-        backend.simpleSearch
+        searchItemStats(backend, dsl)(cfg.fullTextSearch.enabled, fixQuery, itemQuery)
-          .searchSummaryByString(cfg.fullTextSearch.enabled)(fixQuery, itemQuery) match {
-          case Right(summary) =>
-            summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
-          case Left(fail) =>
-            BadRequest(BasicResult(false, fail.render))
-        }
 
       case req @ POST -> Root / "search" =>
         for {
@@ -103,21 +87,7 @@ object ItemRoutes {
             cfg.maxNoteLength
           )
           fixQuery = Query.Fix(user.account, None, None)
-          resp <- backend.simpleSearch
+          resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery)
-            .searchByString(settings)(fixQuery, itemQuery) match {
-            case Right(results) =>
-              val items = results.map(
-                _.fold(
-                  Conversions.mkItemListFts,
-                  Conversions.mkItemListWithTagsFts,
-                  Conversions.mkItemList,
-                  Conversions.mkItemListWithTags
-                )
-              )
-              Ok(items)
-            case Left(fail) =>
-              BadRequest(BasicResult(false, fail.render))
-          }
         } yield resp
 
       case req @ POST -> Root / "searchStats" =>
@@ -125,16 +95,11 @@ object ItemRoutes {
           userQuery <- req.as[ItemQuery]
           itemQuery = ItemQueryString(userQuery.query)
           fixQuery  = Query.Fix(user.account, None, None)
-          resp <- backend.simpleSearch
+          resp <- searchItemStats(backend, dsl)(
-            .searchSummaryByString(cfg.fullTextSearch.enabled)(
+            cfg.fullTextSearch.enabled,
-              fixQuery,
+            fixQuery,
-              itemQuery
+            itemQuery
-            ) match {
+          )
-            case Right(summary) =>
-              summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
-            case Left(fail) =>
-              BadRequest(BasicResult(false, fail.render))
-          }
         } yield resp
 
       //DEPRECATED
@@ -526,6 +491,63 @@ object ItemRoutes {
     }
   }
 
+  def searchItems[F[_]: Sync](
+      backend: BackendApp[F],
+      dsl: Http4sDsl[F]
+  )(settings: OSimpleSearch.Settings, fixQuery: Query.Fix, itemQuery: ItemQueryString) = {
+    import dsl._
+
+    backend.simpleSearch
+      .searchByString(settings)(fixQuery, itemQuery)
+      .flatMap {
+        case StringSearchResult.Success(items) =>
+          Ok(
+            items.fold(
+              Conversions.mkItemListFts,
+              Conversions.mkItemListWithTagsFts,
+              Conversions.mkItemList,
+              Conversions.mkItemListWithTags
+            )
+          )
+        case StringSearchResult.FulltextMismatch(TooMany) =>
+          BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
+        case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
+          BadRequest(
+            BasicResult(
+              false,
+              "Fulltext search must be in root position or inside the first AND."
+            )
+          )
+        case StringSearchResult.ParseFailed(pf) =>
+          BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
+      }
+  }
+
+  def searchItemStats[F[_]: Sync](
+      backend: BackendApp[F],
+      dsl: Http4sDsl[F]
+  )(ftsEnabled: Boolean, fixQuery: Query.Fix, itemQuery: ItemQueryString) = {
+    import dsl._
+    backend.simpleSearch
+      .searchSummaryByString(ftsEnabled)(fixQuery, itemQuery)
+      .flatMap {
+        case StringSearchResult.Success(summary) =>
+          Ok(Conversions.mkSearchStats(summary))
+        case StringSearchResult.FulltextMismatch(TooMany) =>
+          BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
+        case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
+          BadRequest(
+            BasicResult(
+              false,
+              "Fulltext search must be in root position or inside the first AND."
+            )
+          )
+        case StringSearchResult.ParseFailed(pf) =>
+          BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
+      }
+
+  }
+
   implicit final class OptionString(opt: Option[String]) {
     def notEmpty: Option[String] =
       opt.map(_.trim).filter(_.nonEmpty)