mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-04-04 10:29:34 +00:00
Resolve fulltext search queries the same way as before
For now, fulltext search is only possible when being the only term or inside the root AND expression.
This commit is contained in:
Eike Kettner
parent
1c834cbb77
commit
63d146c2de
@ -5,7 +5,7 @@ import cats.implicits._
|
||||
|
||||
import docspell.backend.ops.OSimpleSearch._
|
||||
import docspell.common._
|
||||
import docspell.query.{ItemQueryParser, ParseFailure}
|
||||
import docspell.query._
|
||||
import docspell.store.qb.Batch
|
||||
import docspell.store.queries.Query
|
||||
import docspell.store.queries.SearchSummary
|
||||
@ -20,15 +20,29 @@ trait OSimpleSearch[F[_]] {
|
||||
|
||||
def searchByString(
|
||||
settings: Settings
|
||||
)(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]]
|
||||
)(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]]
|
||||
def searchSummaryByString(
|
||||
useFTS: Boolean
|
||||
)(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]]
|
||||
)(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]]
|
||||
|
||||
}
|
||||
|
||||
object OSimpleSearch {
|
||||
|
||||
sealed trait StringSearchResult[+A]
|
||||
object StringSearchResult {
|
||||
case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing]
|
||||
def parseFailed[A](error: ParseFailure): StringSearchResult[A] =
|
||||
ParseFailed(error)
|
||||
|
||||
case class FulltextMismatch(error: FulltextExtract.FailureResult)
|
||||
extends StringSearchResult[Nothing]
|
||||
def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] =
|
||||
FulltextMismatch(error)
|
||||
|
||||
case class Success[A](value: A) extends StringSearchResult[A]
|
||||
}
|
||||
|
||||
final case class Settings(
|
||||
batch: Batch,
|
||||
useFTS: Boolean,
|
||||
@ -104,19 +118,49 @@ object OSimpleSearch {
|
||||
extends OSimpleSearch[F] {
|
||||
def searchByString(
|
||||
settings: Settings
|
||||
)(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]] =
|
||||
ItemQueryParser
|
||||
.parse(q.query)
|
||||
.map(iq => Query(fix, Query.QueryExpr(iq)))
|
||||
.map(search(settings)(_, None)) //TODO resolve content:xyz expressions
|
||||
)(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[Items]] = {
|
||||
val parsed: Either[StringSearchResult[Items], ItemQuery] =
|
||||
ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed)
|
||||
|
||||
def makeQuery(iq: ItemQuery): F[StringSearchResult[Items]] =
|
||||
iq.findFulltext match {
|
||||
case FulltextExtract.Result.Success(expr, ftq) =>
|
||||
search(settings)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq)
|
||||
.map(StringSearchResult.Success.apply)
|
||||
case other: FulltextExtract.FailureResult =>
|
||||
StringSearchResult.fulltextMismatch[Items](other).pure[F]
|
||||
}
|
||||
|
||||
parsed match {
|
||||
case Right(iq) =>
|
||||
makeQuery(iq)
|
||||
case Left(err) =>
|
||||
err.pure[F]
|
||||
}
|
||||
}
|
||||
|
||||
def searchSummaryByString(
|
||||
useFTS: Boolean
|
||||
)(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] =
|
||||
ItemQueryParser
|
||||
.parse(q.query)
|
||||
.map(iq => Query(fix, Query.QueryExpr(iq)))
|
||||
.map(searchSummary(useFTS)(_, None)) //TODO resolve content:xyz expressions
|
||||
)(fix: Query.Fix, q: ItemQueryString): F[StringSearchResult[SearchSummary]] = {
|
||||
val parsed: Either[StringSearchResult[SearchSummary], ItemQuery] =
|
||||
ItemQueryParser.parse(q.query).leftMap(StringSearchResult.parseFailed)
|
||||
|
||||
def makeQuery(iq: ItemQuery): F[StringSearchResult[SearchSummary]] =
|
||||
iq.findFulltext match {
|
||||
case FulltextExtract.Result.Success(expr, ftq) =>
|
||||
searchSummary(useFTS)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq)
|
||||
.map(StringSearchResult.Success.apply)
|
||||
case other: FulltextExtract.FailureResult =>
|
||||
StringSearchResult.fulltextMismatch[SearchSummary](other).pure[F]
|
||||
}
|
||||
|
||||
parsed match {
|
||||
case Right(iq) =>
|
||||
makeQuery(iq)
|
||||
case Left(err) =>
|
||||
err.pure[F]
|
||||
}
|
||||
}
|
||||
|
||||
def searchSummary(
|
||||
useFTS: Boolean
|
||||
|
||||
@ -0,0 +1,71 @@
|
||||
package docspell.query
|
||||
|
||||
import cats._
|
||||
import cats.implicits._
|
||||
|
||||
import docspell.query.ItemQuery.Expr.AndExpr
|
||||
import docspell.query.ItemQuery.Expr.NotExpr
|
||||
import docspell.query.ItemQuery.Expr.OrExpr
|
||||
import docspell.query.ItemQuery._
|
||||
|
||||
/** Currently, fulltext in a query is only supported when in "root
|
||||
* AND" position
|
||||
*/
|
||||
object FulltextExtract {
|
||||
|
||||
sealed trait Result
|
||||
sealed trait SuccessResult extends Result
|
||||
sealed trait FailureResult extends Result
|
||||
object Result {
|
||||
case class Success(query: Expr, fts: Option[String]) extends SuccessResult
|
||||
case object TooMany extends FailureResult
|
||||
case object UnsupportedPosition extends FailureResult
|
||||
}
|
||||
|
||||
def findFulltext(expr: Expr): Result =
|
||||
lookForFulltext(expr)
|
||||
|
||||
private def lookForFulltext(expr: Expr): Result =
|
||||
expr match {
|
||||
case Expr.Fulltext(ftq) =>
|
||||
Result.Success(ItemQuery.all.expr, ftq.some)
|
||||
case Expr.AndExpr(inner) =>
|
||||
inner.collect({ case Expr.Fulltext(fq) => fq }) match {
|
||||
case Nil =>
|
||||
checkPosition(expr, 0)
|
||||
case e :: Nil =>
|
||||
val c = foldMap(isFulltextExpr)(expr)
|
||||
if (c > 1) Result.TooMany
|
||||
else Result.Success(expr, e.some)
|
||||
case _ =>
|
||||
Result.TooMany
|
||||
}
|
||||
case _ =>
|
||||
checkPosition(expr, 0)
|
||||
}
|
||||
|
||||
private def checkPosition(expr: Expr, max: Int): Result = {
|
||||
val c = foldMap(isFulltextExpr)(expr)
|
||||
if (c > max) Result.UnsupportedPosition
|
||||
else Result.Success(expr, None)
|
||||
}
|
||||
|
||||
private def foldMap[B: Monoid](f: Expr => B)(expr: Expr): B =
|
||||
expr match {
|
||||
case OrExpr(inner) =>
|
||||
inner.map(foldMap(f)).fold
|
||||
case AndExpr(inner) =>
|
||||
inner.map(foldMap(f)).fold
|
||||
case NotExpr(e) =>
|
||||
f(e)
|
||||
case _ =>
|
||||
f(expr)
|
||||
}
|
||||
|
||||
private def isFulltextExpr(expr: Expr): Int =
|
||||
expr match {
|
||||
case Expr.Fulltext(_) => 1
|
||||
case _ => 0
|
||||
}
|
||||
|
||||
}
|
||||
@ -11,7 +11,10 @@ import docspell.query.ItemQuery.Attr.{DateAttr, StringAttr}
|
||||
* against a specific field of an item using some operator or a
|
||||
* combination thereof.
|
||||
*/
|
||||
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String])
|
||||
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) {
|
||||
def findFulltext: FulltextExtract.Result =
|
||||
FulltextExtract.findFulltext(expr)
|
||||
}
|
||||
|
||||
object ItemQuery {
|
||||
val all = ItemQuery(Expr.Exists(Attr.ItemId), Some(""))
|
||||
|
||||
@ -0,0 +1,57 @@
|
||||
package docspell.query
|
||||
|
||||
import cats.implicits._
|
||||
import munit._
|
||||
import docspell.query.FulltextExtract.Result
|
||||
|
||||
class FulltextExtractTest extends FunSuite {
|
||||
|
||||
def findFts(q: String): Result = {
|
||||
val p = ItemQueryParser.parseUnsafe(q)
|
||||
FulltextExtract.findFulltext(p.expr)
|
||||
}
|
||||
|
||||
def assertFts(qstr: String, expect: Result) =
|
||||
assertEquals(findFts(qstr), expect)
|
||||
|
||||
def assertFtsSuccess(qstr: String, expect: Option[String]) = {
|
||||
val q = ItemQueryParser.parseUnsafe(qstr)
|
||||
assertEquals(findFts(qstr), Result.Success(q.expr, expect))
|
||||
}
|
||||
|
||||
test("find fulltext as root") {
|
||||
assertEquals(findFts("content:what"), Result.Success(ItemQuery.all.expr, "what".some))
|
||||
assertEquals(
|
||||
findFts("content:\"what hello\""),
|
||||
Result.Success(ItemQuery.all.expr, "what hello".some)
|
||||
)
|
||||
assertEquals(
|
||||
findFts("content:\"what OR hello\""),
|
||||
Result.Success(ItemQuery.all.expr, "what OR hello".some)
|
||||
)
|
||||
}
|
||||
|
||||
test("find no fulltext") {
|
||||
assertFtsSuccess("name:test", None)
|
||||
}
|
||||
|
||||
test("find fulltext within and") {
|
||||
assertFtsSuccess("content:what name:test", "what".some)
|
||||
assertFtsSuccess("$names:marc* content:what name:test", "what".some)
|
||||
assertFtsSuccess(
|
||||
"$names:marc* date:2021-02 content:\"what else\" name:test",
|
||||
"what else".some
|
||||
)
|
||||
}
|
||||
|
||||
test("too many fulltext searches") {
|
||||
assertFts("content:yes content:no", Result.TooMany)
|
||||
assertFts("content:yes (| name:test content:no)", Result.TooMany)
|
||||
assertFts("content:yes (| name:test (& date:2021-02 content:no))", Result.TooMany)
|
||||
}
|
||||
|
||||
test("wrong fulltext search position") {
|
||||
assertFts("name:test (| date:2021-02 content:yes)", Result.UnsupportedPosition)
|
||||
assertFts("name:test (& date:2021-02 content:yes)", Result.UnsupportedPosition) //TODO
|
||||
}
|
||||
}
|
||||
@ -11,8 +11,11 @@ import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue}
|
||||
import docspell.backend.ops.OFulltext
|
||||
import docspell.backend.ops.OItemSearch.{Batch, Query}
|
||||
import docspell.backend.ops.OSimpleSearch
|
||||
import docspell.backend.ops.OSimpleSearch.StringSearchResult
|
||||
import docspell.common._
|
||||
import docspell.common.syntax.all._
|
||||
import docspell.query.FulltextExtract.Result.TooMany
|
||||
import docspell.query.FulltextExtract.Result.UnsupportedPosition
|
||||
import docspell.restapi.model._
|
||||
import docspell.restserver.Config
|
||||
import docspell.restserver.conv.Conversions
|
||||
@ -60,31 +63,12 @@ object ItemRoutes {
|
||||
cfg.maxNoteLength
|
||||
)
|
||||
val fixQuery = Query.Fix(user.account, None, None)
|
||||
backend.simpleSearch.searchByString(settings)(fixQuery, itemQuery) match {
|
||||
case Right(results) =>
|
||||
val items = results.map(
|
||||
_.fold(
|
||||
Conversions.mkItemListFts,
|
||||
Conversions.mkItemListWithTagsFts,
|
||||
Conversions.mkItemList,
|
||||
Conversions.mkItemListWithTags
|
||||
)
|
||||
)
|
||||
Ok(items)
|
||||
case Left(fail) =>
|
||||
BadRequest(BasicResult(false, fail.render))
|
||||
}
|
||||
searchItems(backend, dsl)(settings, fixQuery, itemQuery)
|
||||
|
||||
case GET -> Root / "searchStats" :? QP.Query(q) =>
|
||||
val itemQuery = ItemQueryString(q)
|
||||
val fixQuery = Query.Fix(user.account, None, None)
|
||||
backend.simpleSearch
|
||||
.searchSummaryByString(cfg.fullTextSearch.enabled)(fixQuery, itemQuery) match {
|
||||
case Right(summary) =>
|
||||
summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
|
||||
case Left(fail) =>
|
||||
BadRequest(BasicResult(false, fail.render))
|
||||
}
|
||||
searchItemStats(backend, dsl)(cfg.fullTextSearch.enabled, fixQuery, itemQuery)
|
||||
|
||||
case req @ POST -> Root / "search" =>
|
||||
for {
|
||||
@ -103,21 +87,7 @@ object ItemRoutes {
|
||||
cfg.maxNoteLength
|
||||
)
|
||||
fixQuery = Query.Fix(user.account, None, None)
|
||||
resp <- backend.simpleSearch
|
||||
.searchByString(settings)(fixQuery, itemQuery) match {
|
||||
case Right(results) =>
|
||||
val items = results.map(
|
||||
_.fold(
|
||||
Conversions.mkItemListFts,
|
||||
Conversions.mkItemListWithTagsFts,
|
||||
Conversions.mkItemList,
|
||||
Conversions.mkItemListWithTags
|
||||
)
|
||||
)
|
||||
Ok(items)
|
||||
case Left(fail) =>
|
||||
BadRequest(BasicResult(false, fail.render))
|
||||
}
|
||||
resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery)
|
||||
} yield resp
|
||||
|
||||
case req @ POST -> Root / "searchStats" =>
|
||||
@ -125,16 +95,11 @@ object ItemRoutes {
|
||||
userQuery <- req.as[ItemQuery]
|
||||
itemQuery = ItemQueryString(userQuery.query)
|
||||
fixQuery = Query.Fix(user.account, None, None)
|
||||
resp <- backend.simpleSearch
|
||||
.searchSummaryByString(cfg.fullTextSearch.enabled)(
|
||||
fixQuery,
|
||||
itemQuery
|
||||
) match {
|
||||
case Right(summary) =>
|
||||
summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
|
||||
case Left(fail) =>
|
||||
BadRequest(BasicResult(false, fail.render))
|
||||
}
|
||||
resp <- searchItemStats(backend, dsl)(
|
||||
cfg.fullTextSearch.enabled,
|
||||
fixQuery,
|
||||
itemQuery
|
||||
)
|
||||
} yield resp
|
||||
|
||||
//DEPRECATED
|
||||
@ -526,6 +491,63 @@ object ItemRoutes {
|
||||
}
|
||||
}
|
||||
|
||||
def searchItems[F[_]: Sync](
|
||||
backend: BackendApp[F],
|
||||
dsl: Http4sDsl[F]
|
||||
)(settings: OSimpleSearch.Settings, fixQuery: Query.Fix, itemQuery: ItemQueryString) = {
|
||||
import dsl._
|
||||
|
||||
backend.simpleSearch
|
||||
.searchByString(settings)(fixQuery, itemQuery)
|
||||
.flatMap {
|
||||
case StringSearchResult.Success(items) =>
|
||||
Ok(
|
||||
items.fold(
|
||||
Conversions.mkItemListFts,
|
||||
Conversions.mkItemListWithTagsFts,
|
||||
Conversions.mkItemList,
|
||||
Conversions.mkItemListWithTags
|
||||
)
|
||||
)
|
||||
case StringSearchResult.FulltextMismatch(TooMany) =>
|
||||
BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
|
||||
case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
|
||||
BadRequest(
|
||||
BasicResult(
|
||||
false,
|
||||
"Fulltext search must be in root position or inside the first AND."
|
||||
)
|
||||
)
|
||||
case StringSearchResult.ParseFailed(pf) =>
|
||||
BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
|
||||
}
|
||||
}
|
||||
|
||||
def searchItemStats[F[_]: Sync](
|
||||
backend: BackendApp[F],
|
||||
dsl: Http4sDsl[F]
|
||||
)(ftsEnabled: Boolean, fixQuery: Query.Fix, itemQuery: ItemQueryString) = {
|
||||
import dsl._
|
||||
backend.simpleSearch
|
||||
.searchSummaryByString(ftsEnabled)(fixQuery, itemQuery)
|
||||
.flatMap {
|
||||
case StringSearchResult.Success(summary) =>
|
||||
Ok(Conversions.mkSearchStats(summary))
|
||||
case StringSearchResult.FulltextMismatch(TooMany) =>
|
||||
BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
|
||||
case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
|
||||
BadRequest(
|
||||
BasicResult(
|
||||
false,
|
||||
"Fulltext search must be in root position or inside the first AND."
|
||||
)
|
||||
)
|
||||
case StringSearchResult.ParseFailed(pf) =>
|
||||
BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
implicit final class OptionString(opt: Option[String]) {
|
||||
def notEmpty: Option[String] =
|
||||
opt.map(_.trim).filter(_.nonEmpty)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user