diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala index 3a822eba..34641cb5 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala @@ -10,7 +10,7 @@ import java.time.LocalDate import cats.effect._ import cats.syntax.all._ -import cats.~> +import cats.{Functor, ~>} import fs2.Stream import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary} @@ -52,9 +52,23 @@ trait OSearch[F[_]] { fulltextQuery: Option[String] ): F[Vector[ListItemWithTags]] + /** Selects either `search` or `searchWithDetails`. For the former the items are filled + * with empty details. + */ + final def searchSelect( + withDetails: Boolean, + maxNoteLen: Int, + today: LocalDate, + batch: Batch + )( + q: Query, + fulltextQuery: Option[String] + )(implicit F: Functor[F]): F[Vector[ListItemWithTags]] = + if (withDetails) searchWithDetails(maxNoteLen, today, batch)(q, fulltextQuery) + else search(maxNoteLen, today, batch)(q, fulltextQuery).map(_.map(_.toWithTags)) + /** Run multiple database calls with the give query to collect a summary. */ def searchSummary( - mode: SearchMode, today: LocalDate )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] @@ -80,41 +94,49 @@ object OSearch { accountId: AccountId, mode: SearchMode, qs: String - ): QueryParseResult = - ItemQueryParser.parse(qs) match { - case Right(iq) => - val validItemQuery = - mode match { - case SearchMode.Trashed => ItemQuery.Expr.Trashed - case SearchMode.Normal => ItemQuery.Expr.ValidItemStates - case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed + ): QueryParseResult = { + val validItemQuery = + mode match { + case SearchMode.Trashed => ItemQuery.Expr.Trashed + case SearchMode.Normal => ItemQuery.Expr.ValidItemStates + case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed + } + + if (qs.trim.isEmpty) { + val qf = Query.Fix(accountId, Some(validItemQuery), None) + val qq = Query.QueryExpr(None) + val q = Query(qf, qq) + QueryParseResult.Success(q, None) + } else + ItemQueryParser.parse(qs) match { + case Right(iq) => + FulltextExtract.findFulltext(iq.expr) match { + case FulltextExtract.Result.SuccessNoFulltext(expr) => + val qf = Query.Fix(accountId, Some(validItemQuery), None) + val qq = Query.QueryExpr(expr) + val q = Query(qf, qq) + QueryParseResult.Success(q, None) + + case FulltextExtract.Result.SuccessNoExpr(fts) => + val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore)) + val qq = Query.QueryExpr(None) + val q = Query(qf, qq) + QueryParseResult.Success(q, Some(fts)) + + case FulltextExtract.Result.SuccessBoth(expr, fts) => + val qf = Query.Fix(accountId, Some(validItemQuery), None) + val qq = Query.QueryExpr(expr) + val q = Query(qf, qq) + QueryParseResult.Success(q, Some(fts)) + + case f: FulltextExtract.FailureResult => + QueryParseResult.FulltextMismatch(f) } - FulltextExtract.findFulltext(iq.expr) match { - case FulltextExtract.Result.SuccessNoFulltext(expr) => - val qf = Query.Fix(accountId, Some(validItemQuery), None) - val qq = Query.QueryExpr(expr) - val q = Query(qf, qq) - QueryParseResult.Success(q, None) - case FulltextExtract.Result.SuccessNoExpr(fts) => - val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore)) - val qq = Query.QueryExpr(None) - val q = Query(qf, qq) - QueryParseResult.Success(q, Some(fts)) - - case FulltextExtract.Result.SuccessBoth(expr, fts) => - val qf = Query.Fix(accountId, Some(validItemQuery), None) - val qq = Query.QueryExpr(expr) - val q = Query(qf, qq) - QueryParseResult.Success(q, Some(fts)) - - case f: FulltextExtract.FailureResult => - QueryParseResult.FulltextMismatch(f) - } - - case Left(err) => - QueryParseResult.ParseFailed(err).cast - } + case Left(err) => + QueryParseResult.ParseFailed(err).cast + } + } def search(maxNoteLen: Int, today: LocalDate, batch: Batch)( q: Query, @@ -167,7 +189,6 @@ object OSearch { } yield resolved def searchSummary( - mode: SearchMode, today: LocalDate )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = fulltextQuery match { @@ -194,7 +215,7 @@ object OSearch { store .transact(QFolder.getMemberFolders(account)) .map(folders => - FtsQuery(ftq, account.collective, batch.limit, batch.offset) + FtsQuery(ftq, account.collective, batch.limit, 0) .withFolders(folders) ) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 28421776..ce587fbd 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -297,7 +297,7 @@ trait Conversions { relatedItems = i.relatedItems ) - private def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight = + def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight = AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount) def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index f77ac5a2..acf1e63a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -41,10 +41,11 @@ object ItemRoutes { user: AuthToken ): HttpRoutes[F] = { val logger = docspell.logging.getLogger[F] + val searchPart = ItemSearchPart[F](backend, cfg, user) val dsl = new Http4sDsl[F] {} import dsl._ - ItemSearchPart(backend, cfg, user) <+> + searchPart.routes <+> HttpRoutes.of { case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( offset diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala index 7546a63b..c9603b3d 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala @@ -6,50 +6,206 @@ package docspell.restserver.routes -import cats.effect.Async +import java.time.LocalDate + +import cats.effect._ +import cats.syntax.all._ import docspell.backend.BackendApp import docspell.backend.auth.AuthToken +import docspell.backend.ops.search.QueryParseResult +import docspell.common.{SearchMode, Timestamp} +import docspell.query.FulltextExtract +import docspell.restapi.model._ import docspell.restserver.Config +import docspell.restserver.conv.Conversions import docspell.restserver.http4s.{QueryParam => QP} +import docspell.store.qb.Batch +import docspell.store.queries.ListItemWithTags -import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ import org.http4s.dsl.Http4sDsl +import org.http4s.{HttpRoutes, Response} + +final class ItemSearchPart[F[_]: Async]( + backend: BackendApp[F], + cfg: Config, + authToken: AuthToken +) extends Http4sDsl[F] { + + private[this] val logger = docspell.logging.getLogger[F] + + def routes: HttpRoutes[F] = + if (!cfg.featureSearch2) HttpRoutes.empty + else + HttpRoutes.of { + case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( + offset + ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => + val userQuery = + ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse("")) + + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap(search(userQuery, _)) + + case req @ POST -> Root / "search" => + for { + userQuery <- req.as[ItemQuery] + today <- Timestamp.current[F] + resp <- search(userQuery, today.toUtcDate) + } yield resp + + case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => + val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse("")) + Timestamp + .current[F] + .map(_.toUtcDate) + .flatMap(searchStats(userQuery, _)) + + case req @ POST -> Root / "searchStats" => + for { + userQuery <- req.as[ItemQuery] + today <- Timestamp.current[F].map(_.toUtcDate) + resp <- searchStats(userQuery, today) + } yield resp + } + + def searchStats(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = { + val mode = userQuery.searchMode.getOrElse(SearchMode.Normal) + parsedQuery(userQuery, mode) + .fold( + identity, + res => + for { + summary <- backend.search.searchSummary(today)(res.q, res.ftq) + resp <- Ok(Conversions.mkSearchStats(summary)) + } yield resp + ) + } + + def search(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = { + val details = userQuery.withDetails.getOrElse(false) + val batch = + Batch(userQuery.offset.getOrElse(0), userQuery.limit.getOrElse(cfg.maxItemPageSize)) + .restrictLimitTo(cfg.maxItemPageSize) + val limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize) + val mode = userQuery.searchMode.getOrElse(SearchMode.Normal) + + parsedQuery(userQuery, mode) + .fold( + identity, + res => + for { + items <- backend.search + .searchSelect(details, cfg.maxNoteLength, today, batch)( + res.q, + res.ftq + ) + + // order is always by date unless q is empty and ftq is not + // TODO this is not obvious from the types and an impl detail. + ftsOrder = res.q.cond.isEmpty && res.ftq.isDefined + + resp <- Ok(convert(items, batch, limitCapped, ftsOrder)) + } yield resp + ) + } + + def parsedQuery( + userQuery: ItemQuery, + mode: SearchMode + ): Either[F[Response[F]], QueryParseResult.Success] = + backend.search.parseQueryString(authToken.account, mode, userQuery.query) match { + case s: QueryParseResult.Success => + Right(s) + + case QueryParseResult.ParseFailed(err) => + Left(BadRequest(BasicResult(false, s"Invalid query: $err"))) + + case QueryParseResult.FulltextMismatch(FulltextExtract.Result.TooMany) => + Left( + BadRequest( + BasicResult(false, "Only one fulltext search expression is allowed.") + ) + ) + case QueryParseResult.FulltextMismatch( + FulltextExtract.Result.UnsupportedPosition + ) => + Left( + BadRequest( + BasicResult( + false, + "A fulltext search may only appear in the root and expression." + ) + ) + ) + } + + def convert( + items: Vector[ListItemWithTags], + batch: Batch, + capped: Boolean, + ftsOrder: Boolean + ): ItemLightList = + if (ftsOrder) + ItemLightList( + List(ItemLightGroup("Results", items.map(convertItem).toList)), + batch.limit, + batch.offset, + capped + ) + else { + val groups = items.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7)) + + def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup = + ItemLightGroup(g._1, g._2.map(convertItem).toList) + + val gs = + groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) + + ItemLightList(gs, batch.limit, batch.offset, capped) + } + + def convertItem(item: ListItemWithTags): ItemLight = + ItemLight( + id = item.item.id, + name = item.item.name, + state = item.item.state, + date = item.item.date, + dueDate = item.item.dueDate, + source = item.item.source, + direction = item.item.direction.name.some, + corrOrg = item.item.corrOrg.map(Conversions.mkIdName), + corrPerson = item.item.corrPerson.map(Conversions.mkIdName), + concPerson = item.item.concPerson.map(Conversions.mkIdName), + concEquipment = item.item.concEquip.map(Conversions.mkIdName), + folder = item.item.folder.map(Conversions.mkIdName), + attachments = item.attachments.map(Conversions.mkAttachmentLight), + tags = item.tags.map(Conversions.mkTag), + customfields = item.customfields.map(Conversions.mkItemFieldValue), + relatedItems = item.relatedItems, + notes = item.item.notes, + highlighting = item.item.decodeContext match { + case Some(Right(hlctx)) => + hlctx.map(c => HighlightEntry(c.name, c.context)) + case Some(Left(err)) => + logger.asUnsafe.error( + s"Internal error: cannot decode highlight context '${item.item.context}': $err" + ) + Nil + case None => + Nil + } + ) +} -@annotation.nowarn object ItemSearchPart { - def apply[F[_]: Async]( backend: BackendApp[F], cfg: Config, - authToken: AuthToken - ): HttpRoutes[F] = - if (cfg.featureSearch2) routes(backend, cfg, authToken) - else HttpRoutes.empty - - def routes[F[_]: Async]( - backend: BackendApp[F], - cfg: Config, - authToken: AuthToken - ): HttpRoutes[F] = { - val logger = docspell.logging.getLogger[F] - val dsl = new Http4sDsl[F] {} - import dsl._ - - HttpRoutes.of { - case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( - offset - ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => - ??? - - case req @ POST -> Root / "search" => - ??? - - case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => - ??? - - case req @ POST -> Root / "searchStats" => - ??? - } - } + token: AuthToken + ): ItemSearchPart[F] = + new ItemSearchPart[F](backend, cfg, token) } diff --git a/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala b/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala index 8f801df9..62646f28 100644 --- a/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala +++ b/modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala @@ -23,7 +23,7 @@ trait FtsSupport { val tt = cteTable(ftst) select .appendCte(ftst.distinctCteSimple(tt.tableName)) - .changeFrom(_.innerJoin(tt, itemTable.id === tt.id)) + .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id))) case None => select } @@ -37,7 +37,19 @@ trait FtsSupport { val tt = cteTable(ftst) select .appendCte(ftst.distinctCte(tt.tableName)) - .changeFrom(_.innerJoin(tt, itemTable.id === tt.id)) + .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id))) + case None => + select + } + + def ftsCondition( + itemTable: RItem.Table, + ftsTable: Option[TempFtsTable.Table] + ): Select = + ftsTable match { + case Some(ftst) => + val ftsIds = Select(ftst.id.s, from(ftst)).distinct + select.changeWhere(c => c && itemTable.id.in(ftsIds)) case None => select } diff --git a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala index bd0a9743..ffac6fd2 100644 --- a/modules/store/src/main/scala/docspell/store/queries/ListItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/ListItem.scala @@ -29,7 +29,7 @@ case class ListItem( def decodeContext: Option[Either[String, List[ContextEntry]]] = context.map(_.trim).filter(_.nonEmpty).map { str => - // This is a bit… well. The common denominator for the dbms used is string aggregation + // This is a bit…. The common denominator for the dbms used is string aggregation // when combining multiple matches. So the `ContextEntry` objects are concatenated and // separated by comma. TemplateFtsTable ensures than the single entries are all json // objects. @@ -40,4 +40,7 @@ case class ListItem( .map(_.getMessage) .map(_.flatten) } + + def toWithTags: ListItemWithTags = + ListItemWithTags(this, Nil, Nil, Nil, Nil) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index f6f310d6..a64664cb 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -475,9 +475,9 @@ object QItem extends FtsSupport { val base = findItemsBase(q.fix, today, 0, None).unwrap - .joinFtsIdOnly(i, ftsTable) .changeFrom(_.prepend(fieldJoin)) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) + .ftsCondition(i, ftsTable) .groupBy(GroupBy(cf.all)) val basicFields = Nel.of(