From e47396182d0bb9db0642db766e50e60e3bb884b0 Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Tue, 31 May 2022 19:56:45 +0200
Subject: [PATCH] Fix obvious things and add search summary
---
.../docspell/backend/ops/search/OSearch.scala | 95 +++++---
.../restserver/conv/Conversions.scala | 2 +-
.../restserver/routes/ItemRoutes.scala | 3 +-
.../restserver/routes/ItemSearchPart.scala | 224 +++++++++++++++---
.../docspell/store/queries/FtsSupport.scala | 16 +-
.../docspell/store/queries/ListItem.scala | 5 +-
.../scala/docspell/store/queries/QItem.scala | 2 +-
7 files changed, 270 insertions(+), 77 deletions(-)
diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala
index 3a822eba..34641cb5 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala
@@ -10,7 +10,7 @@ import java.time.LocalDate
import cats.effect._
import cats.syntax.all._
-import cats.~>
+import cats.{Functor, ~>}
import fs2.Stream
import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary}
@@ -52,9 +52,23 @@ trait OSearch[F[_]] {
fulltextQuery: Option[String]
): F[Vector[ListItemWithTags]]
+ /** Selects either `search` or `searchWithDetails`. For the former the items are filled
+ * with empty details.
+ */
+ final def searchSelect(
+ withDetails: Boolean,
+ maxNoteLen: Int,
+ today: LocalDate,
+ batch: Batch
+ )(
+ q: Query,
+ fulltextQuery: Option[String]
+ )(implicit F: Functor[F]): F[Vector[ListItemWithTags]] =
+ if (withDetails) searchWithDetails(maxNoteLen, today, batch)(q, fulltextQuery)
+ else search(maxNoteLen, today, batch)(q, fulltextQuery).map(_.map(_.toWithTags))
+
/** Run multiple database calls with the give query to collect a summary. */
def searchSummary(
- mode: SearchMode,
today: LocalDate
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
@@ -80,41 +94,49 @@ object OSearch {
accountId: AccountId,
mode: SearchMode,
qs: String
- ): QueryParseResult =
- ItemQueryParser.parse(qs) match {
- case Right(iq) =>
- val validItemQuery =
- mode match {
- case SearchMode.Trashed => ItemQuery.Expr.Trashed
- case SearchMode.Normal => ItemQuery.Expr.ValidItemStates
- case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed
+ ): QueryParseResult = {
+ val validItemQuery =
+ mode match {
+ case SearchMode.Trashed => ItemQuery.Expr.Trashed
+ case SearchMode.Normal => ItemQuery.Expr.ValidItemStates
+ case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed
+ }
+
+ if (qs.trim.isEmpty) {
+ val qf = Query.Fix(accountId, Some(validItemQuery), None)
+ val qq = Query.QueryExpr(None)
+ val q = Query(qf, qq)
+ QueryParseResult.Success(q, None)
+ } else
+ ItemQueryParser.parse(qs) match {
+ case Right(iq) =>
+ FulltextExtract.findFulltext(iq.expr) match {
+ case FulltextExtract.Result.SuccessNoFulltext(expr) =>
+ val qf = Query.Fix(accountId, Some(validItemQuery), None)
+ val qq = Query.QueryExpr(expr)
+ val q = Query(qf, qq)
+ QueryParseResult.Success(q, None)
+
+ case FulltextExtract.Result.SuccessNoExpr(fts) =>
+ val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
+ val qq = Query.QueryExpr(None)
+ val q = Query(qf, qq)
+ QueryParseResult.Success(q, Some(fts))
+
+ case FulltextExtract.Result.SuccessBoth(expr, fts) =>
+ val qf = Query.Fix(accountId, Some(validItemQuery), None)
+ val qq = Query.QueryExpr(expr)
+ val q = Query(qf, qq)
+ QueryParseResult.Success(q, Some(fts))
+
+ case f: FulltextExtract.FailureResult =>
+ QueryParseResult.FulltextMismatch(f)
}
- FulltextExtract.findFulltext(iq.expr) match {
- case FulltextExtract.Result.SuccessNoFulltext(expr) =>
- val qf = Query.Fix(accountId, Some(validItemQuery), None)
- val qq = Query.QueryExpr(expr)
- val q = Query(qf, qq)
- QueryParseResult.Success(q, None)
- case FulltextExtract.Result.SuccessNoExpr(fts) =>
- val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
- val qq = Query.QueryExpr(None)
- val q = Query(qf, qq)
- QueryParseResult.Success(q, Some(fts))
-
- case FulltextExtract.Result.SuccessBoth(expr, fts) =>
- val qf = Query.Fix(accountId, Some(validItemQuery), None)
- val qq = Query.QueryExpr(expr)
- val q = Query(qf, qq)
- QueryParseResult.Success(q, Some(fts))
-
- case f: FulltextExtract.FailureResult =>
- QueryParseResult.FulltextMismatch(f)
- }
-
- case Left(err) =>
- QueryParseResult.ParseFailed(err).cast
- }
+ case Left(err) =>
+ QueryParseResult.ParseFailed(err).cast
+ }
+ }
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
q: Query,
@@ -167,7 +189,6 @@ object OSearch {
} yield resolved
def searchSummary(
- mode: SearchMode,
today: LocalDate
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
fulltextQuery match {
@@ -194,7 +215,7 @@ object OSearch {
store
.transact(QFolder.getMemberFolders(account))
.map(folders =>
- FtsQuery(ftq, account.collective, batch.limit, batch.offset)
+ FtsQuery(ftq, account.collective, batch.limit, 0)
.withFolders(folders)
)
diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
index 28421776..ce587fbd 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@@ -297,7 +297,7 @@ trait Conversions {
relatedItems = i.relatedItems
)
- private def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
+ def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount)
def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = {
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
index f77ac5a2..acf1e63a 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
@@ -41,10 +41,11 @@ object ItemRoutes {
user: AuthToken
): HttpRoutes[F] = {
val logger = docspell.logging.getLogger[F]
+ val searchPart = ItemSearchPart[F](backend, cfg, user)
val dsl = new Http4sDsl[F] {}
import dsl._
- ItemSearchPart(backend, cfg, user) <+>
+ searchPart.routes <+>
HttpRoutes.of {
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
offset
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala
index 7546a63b..c9603b3d 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala
@@ -6,50 +6,206 @@
package docspell.restserver.routes
-import cats.effect.Async
+import java.time.LocalDate
+
+import cats.effect._
+import cats.syntax.all._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
+import docspell.backend.ops.search.QueryParseResult
+import docspell.common.{SearchMode, Timestamp}
+import docspell.query.FulltextExtract
+import docspell.restapi.model._
import docspell.restserver.Config
+import docspell.restserver.conv.Conversions
import docspell.restserver.http4s.{QueryParam => QP}
+import docspell.store.qb.Batch
+import docspell.store.queries.ListItemWithTags
-import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.Http4sDsl
+import org.http4s.{HttpRoutes, Response}
+
+final class ItemSearchPart[F[_]: Async](
+ backend: BackendApp[F],
+ cfg: Config,
+ authToken: AuthToken
+) extends Http4sDsl[F] {
+
+ private[this] val logger = docspell.logging.getLogger[F]
+
+ def routes: HttpRoutes[F] =
+ if (!cfg.featureSearch2) HttpRoutes.empty
+ else
+ HttpRoutes.of {
+ case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
+ offset
+ ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
+ val userQuery =
+ ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse(""))
+
+ Timestamp
+ .current[F]
+ .map(_.toUtcDate)
+ .flatMap(search(userQuery, _))
+
+ case req @ POST -> Root / "search" =>
+ for {
+ userQuery <- req.as[ItemQuery]
+ today <- Timestamp.current[F]
+ resp <- search(userQuery, today.toUtcDate)
+ } yield resp
+
+ case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
+ val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse(""))
+ Timestamp
+ .current[F]
+ .map(_.toUtcDate)
+ .flatMap(searchStats(userQuery, _))
+
+ case req @ POST -> Root / "searchStats" =>
+ for {
+ userQuery <- req.as[ItemQuery]
+ today <- Timestamp.current[F].map(_.toUtcDate)
+ resp <- searchStats(userQuery, today)
+ } yield resp
+ }
+
+ def searchStats(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
+ val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+ parsedQuery(userQuery, mode)
+ .fold(
+ identity,
+ res =>
+ for {
+ summary <- backend.search.searchSummary(today)(res.q, res.ftq)
+ resp <- Ok(Conversions.mkSearchStats(summary))
+ } yield resp
+ )
+ }
+
+ def search(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
+ val details = userQuery.withDetails.getOrElse(false)
+ val batch =
+ Batch(userQuery.offset.getOrElse(0), userQuery.limit.getOrElse(cfg.maxItemPageSize))
+ .restrictLimitTo(cfg.maxItemPageSize)
+ val limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
+ val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+
+ parsedQuery(userQuery, mode)
+ .fold(
+ identity,
+ res =>
+ for {
+ items <- backend.search
+ .searchSelect(details, cfg.maxNoteLength, today, batch)(
+ res.q,
+ res.ftq
+ )
+
+ // order is always by date unless q is empty and ftq is not
+ // TODO this is not obvious from the types and an impl detail.
+ ftsOrder = res.q.cond.isEmpty && res.ftq.isDefined
+
+ resp <- Ok(convert(items, batch, limitCapped, ftsOrder))
+ } yield resp
+ )
+ }
+
+ def parsedQuery(
+ userQuery: ItemQuery,
+ mode: SearchMode
+ ): Either[F[Response[F]], QueryParseResult.Success] =
+ backend.search.parseQueryString(authToken.account, mode, userQuery.query) match {
+ case s: QueryParseResult.Success =>
+ Right(s)
+
+ case QueryParseResult.ParseFailed(err) =>
+ Left(BadRequest(BasicResult(false, s"Invalid query: $err")))
+
+ case QueryParseResult.FulltextMismatch(FulltextExtract.Result.TooMany) =>
+ Left(
+ BadRequest(
+ BasicResult(false, "Only one fulltext search expression is allowed.")
+ )
+ )
+ case QueryParseResult.FulltextMismatch(
+ FulltextExtract.Result.UnsupportedPosition
+ ) =>
+ Left(
+ BadRequest(
+ BasicResult(
+ false,
+ "A fulltext search may only appear in the root and expression."
+ )
+ )
+ )
+ }
+
+ def convert(
+ items: Vector[ListItemWithTags],
+ batch: Batch,
+ capped: Boolean,
+ ftsOrder: Boolean
+ ): ItemLightList =
+ if (ftsOrder)
+ ItemLightList(
+ List(ItemLightGroup("Results", items.map(convertItem).toList)),
+ batch.limit,
+ batch.offset,
+ capped
+ )
+ else {
+ val groups = items.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
+
+ def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup =
+ ItemLightGroup(g._1, g._2.map(convertItem).toList)
+
+ val gs =
+ groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
+
+ ItemLightList(gs, batch.limit, batch.offset, capped)
+ }
+
+ def convertItem(item: ListItemWithTags): ItemLight =
+ ItemLight(
+ id = item.item.id,
+ name = item.item.name,
+ state = item.item.state,
+ date = item.item.date,
+ dueDate = item.item.dueDate,
+ source = item.item.source,
+ direction = item.item.direction.name.some,
+ corrOrg = item.item.corrOrg.map(Conversions.mkIdName),
+ corrPerson = item.item.corrPerson.map(Conversions.mkIdName),
+ concPerson = item.item.concPerson.map(Conversions.mkIdName),
+ concEquipment = item.item.concEquip.map(Conversions.mkIdName),
+ folder = item.item.folder.map(Conversions.mkIdName),
+ attachments = item.attachments.map(Conversions.mkAttachmentLight),
+ tags = item.tags.map(Conversions.mkTag),
+ customfields = item.customfields.map(Conversions.mkItemFieldValue),
+ relatedItems = item.relatedItems,
+ notes = item.item.notes,
+ highlighting = item.item.decodeContext match {
+ case Some(Right(hlctx)) =>
+ hlctx.map(c => HighlightEntry(c.name, c.context))
+ case Some(Left(err)) =>
+ logger.asUnsafe.error(
+ s"Internal error: cannot decode highlight context '${item.item.context}': $err"
+ )
+ Nil
+ case None =>
+ Nil
+ }
+ )
+}
-@annotation.nowarn
object ItemSearchPart {
-
def apply[F[_]: Async](
backend: BackendApp[F],
cfg: Config,
- authToken: AuthToken
- ): HttpRoutes[F] =
- if (cfg.featureSearch2) routes(backend, cfg, authToken)
- else HttpRoutes.empty
-
- def routes[F[_]: Async](
- backend: BackendApp[F],
- cfg: Config,
- authToken: AuthToken
- ): HttpRoutes[F] = {
- val logger = docspell.logging.getLogger[F]
- val dsl = new Http4sDsl[F] {}
- import dsl._
-
- HttpRoutes.of {
- case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
- offset
- ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
- ???
-
- case req @ POST -> Root / "search" =>
- ???
-
- case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
- ???
-
- case req @ POST -> Root / "searchStats" =>
- ???
- }
- }
+ token: AuthToken
+ ): ItemSearchPart[F] =
+ new ItemSearchPart[F](backend, cfg, token)
}
diff --git a/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala b/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala
index 8f801df9..62646f28 100644
--- a/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala
@@ -23,7 +23,7 @@ trait FtsSupport {
val tt = cteTable(ftst)
select
.appendCte(ftst.distinctCteSimple(tt.tableName))
- .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+ .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
case None =>
select
}
@@ -37,7 +37,19 @@ trait FtsSupport {
val tt = cteTable(ftst)
select
.appendCte(ftst.distinctCte(tt.tableName))
- .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+ .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
+ case None =>
+ select
+ }
+
+ def ftsCondition(
+ itemTable: RItem.Table,
+ ftsTable: Option[TempFtsTable.Table]
+ ): Select =
+ ftsTable match {
+ case Some(ftst) =>
+ val ftsIds = Select(ftst.id.s, from(ftst)).distinct
+ select.changeWhere(c => c && itemTable.id.in(ftsIds))
case None =>
select
}
diff --git a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala
index bd0a9743..ffac6fd2 100644
--- a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala
@@ -29,7 +29,7 @@ case class ListItem(
def decodeContext: Option[Either[String, List[ContextEntry]]] =
context.map(_.trim).filter(_.nonEmpty).map { str =>
- // This is a bit… well. The common denominator for the dbms used is string aggregation
+ // This is a bit…. The common denominator for the dbms used is string aggregation
// when combining multiple matches. So the `ContextEntry` objects are concatenated and
// separated by comma. TemplateFtsTable ensures than the single entries are all json
// objects.
@@ -40,4 +40,7 @@ case class ListItem(
.map(_.getMessage)
.map(_.flatten)
}
+
+ def toWithTags: ListItemWithTags =
+ ListItemWithTags(this, Nil, Nil, Nil, Nil)
}
diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala
index f6f310d6..a64664cb 100644
--- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala
@@ -475,9 +475,9 @@ object QItem extends FtsSupport {
val base =
findItemsBase(q.fix, today, 0, None).unwrap
- .joinFtsIdOnly(i, ftsTable)
.changeFrom(_.prepend(fieldJoin))
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
+ .ftsCondition(i, ftsTable)
.groupBy(GroupBy(cf.all))
val basicFields = Nel.of(