mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-04-04 10:29:34 +00:00
Fix obvious things and add search summary
This commit is contained in:
eikek
parent
1266cdefe1
commit
e47396182d
@ -10,7 +10,7 @@ import java.time.LocalDate
|
||||
|
||||
import cats.effect._
|
||||
import cats.syntax.all._
|
||||
import cats.~>
|
||||
import cats.{Functor, ~>}
|
||||
import fs2.Stream
|
||||
|
||||
import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary}
|
||||
@ -52,9 +52,23 @@ trait OSearch[F[_]] {
|
||||
fulltextQuery: Option[String]
|
||||
): F[Vector[ListItemWithTags]]
|
||||
|
||||
/** Selects either `search` or `searchWithDetails`. For the former the items are filled
|
||||
* with empty details.
|
||||
*/
|
||||
final def searchSelect(
|
||||
withDetails: Boolean,
|
||||
maxNoteLen: Int,
|
||||
today: LocalDate,
|
||||
batch: Batch
|
||||
)(
|
||||
q: Query,
|
||||
fulltextQuery: Option[String]
|
||||
)(implicit F: Functor[F]): F[Vector[ListItemWithTags]] =
|
||||
if (withDetails) searchWithDetails(maxNoteLen, today, batch)(q, fulltextQuery)
|
||||
else search(maxNoteLen, today, batch)(q, fulltextQuery).map(_.map(_.toWithTags))
|
||||
|
||||
/** Run multiple database calls with the give query to collect a summary. */
|
||||
def searchSummary(
|
||||
mode: SearchMode,
|
||||
today: LocalDate
|
||||
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
|
||||
|
||||
@ -80,41 +94,49 @@ object OSearch {
|
||||
accountId: AccountId,
|
||||
mode: SearchMode,
|
||||
qs: String
|
||||
): QueryParseResult =
|
||||
ItemQueryParser.parse(qs) match {
|
||||
case Right(iq) =>
|
||||
val validItemQuery =
|
||||
mode match {
|
||||
case SearchMode.Trashed => ItemQuery.Expr.Trashed
|
||||
case SearchMode.Normal => ItemQuery.Expr.ValidItemStates
|
||||
case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed
|
||||
): QueryParseResult = {
|
||||
val validItemQuery =
|
||||
mode match {
|
||||
case SearchMode.Trashed => ItemQuery.Expr.Trashed
|
||||
case SearchMode.Normal => ItemQuery.Expr.ValidItemStates
|
||||
case SearchMode.All => ItemQuery.Expr.ValidItemsOrTrashed
|
||||
}
|
||||
|
||||
if (qs.trim.isEmpty) {
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), None)
|
||||
val qq = Query.QueryExpr(None)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, None)
|
||||
} else
|
||||
ItemQueryParser.parse(qs) match {
|
||||
case Right(iq) =>
|
||||
FulltextExtract.findFulltext(iq.expr) match {
|
||||
case FulltextExtract.Result.SuccessNoFulltext(expr) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), None)
|
||||
val qq = Query.QueryExpr(expr)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, None)
|
||||
|
||||
case FulltextExtract.Result.SuccessNoExpr(fts) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
|
||||
val qq = Query.QueryExpr(None)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, Some(fts))
|
||||
|
||||
case FulltextExtract.Result.SuccessBoth(expr, fts) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), None)
|
||||
val qq = Query.QueryExpr(expr)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, Some(fts))
|
||||
|
||||
case f: FulltextExtract.FailureResult =>
|
||||
QueryParseResult.FulltextMismatch(f)
|
||||
}
|
||||
FulltextExtract.findFulltext(iq.expr) match {
|
||||
case FulltextExtract.Result.SuccessNoFulltext(expr) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), None)
|
||||
val qq = Query.QueryExpr(expr)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, None)
|
||||
|
||||
case FulltextExtract.Result.SuccessNoExpr(fts) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
|
||||
val qq = Query.QueryExpr(None)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, Some(fts))
|
||||
|
||||
case FulltextExtract.Result.SuccessBoth(expr, fts) =>
|
||||
val qf = Query.Fix(accountId, Some(validItemQuery), None)
|
||||
val qq = Query.QueryExpr(expr)
|
||||
val q = Query(qf, qq)
|
||||
QueryParseResult.Success(q, Some(fts))
|
||||
|
||||
case f: FulltextExtract.FailureResult =>
|
||||
QueryParseResult.FulltextMismatch(f)
|
||||
}
|
||||
|
||||
case Left(err) =>
|
||||
QueryParseResult.ParseFailed(err).cast
|
||||
}
|
||||
case Left(err) =>
|
||||
QueryParseResult.ParseFailed(err).cast
|
||||
}
|
||||
}
|
||||
|
||||
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
|
||||
q: Query,
|
||||
@ -167,7 +189,6 @@ object OSearch {
|
||||
} yield resolved
|
||||
|
||||
def searchSummary(
|
||||
mode: SearchMode,
|
||||
today: LocalDate
|
||||
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
|
||||
fulltextQuery match {
|
||||
@ -194,7 +215,7 @@ object OSearch {
|
||||
store
|
||||
.transact(QFolder.getMemberFolders(account))
|
||||
.map(folders =>
|
||||
FtsQuery(ftq, account.collective, batch.limit, batch.offset)
|
||||
FtsQuery(ftq, account.collective, batch.limit, 0)
|
||||
.withFolders(folders)
|
||||
)
|
||||
|
||||
|
||||
@ -297,7 +297,7 @@ trait Conversions {
|
||||
relatedItems = i.relatedItems
|
||||
)
|
||||
|
||||
private def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
|
||||
def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
|
||||
AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount)
|
||||
|
||||
def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = {
|
||||
|
||||
@ -41,10 +41,11 @@ object ItemRoutes {
|
||||
user: AuthToken
|
||||
): HttpRoutes[F] = {
|
||||
val logger = docspell.logging.getLogger[F]
|
||||
val searchPart = ItemSearchPart[F](backend, cfg, user)
|
||||
val dsl = new Http4sDsl[F] {}
|
||||
import dsl._
|
||||
|
||||
ItemSearchPart(backend, cfg, user) <+>
|
||||
searchPart.routes <+>
|
||||
HttpRoutes.of {
|
||||
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
|
||||
offset
|
||||
|
||||
@ -6,50 +6,206 @@
|
||||
|
||||
package docspell.restserver.routes
|
||||
|
||||
import cats.effect.Async
|
||||
import java.time.LocalDate
|
||||
|
||||
import cats.effect._
|
||||
import cats.syntax.all._
|
||||
|
||||
import docspell.backend.BackendApp
|
||||
import docspell.backend.auth.AuthToken
|
||||
import docspell.backend.ops.search.QueryParseResult
|
||||
import docspell.common.{SearchMode, Timestamp}
|
||||
import docspell.query.FulltextExtract
|
||||
import docspell.restapi.model._
|
||||
import docspell.restserver.Config
|
||||
import docspell.restserver.conv.Conversions
|
||||
import docspell.restserver.http4s.{QueryParam => QP}
|
||||
import docspell.store.qb.Batch
|
||||
import docspell.store.queries.ListItemWithTags
|
||||
|
||||
import org.http4s.HttpRoutes
|
||||
import org.http4s.circe.CirceEntityCodec._
|
||||
import org.http4s.dsl.Http4sDsl
|
||||
import org.http4s.{HttpRoutes, Response}
|
||||
|
||||
final class ItemSearchPart[F[_]: Async](
|
||||
backend: BackendApp[F],
|
||||
cfg: Config,
|
||||
authToken: AuthToken
|
||||
) extends Http4sDsl[F] {
|
||||
|
||||
private[this] val logger = docspell.logging.getLogger[F]
|
||||
|
||||
def routes: HttpRoutes[F] =
|
||||
if (!cfg.featureSearch2) HttpRoutes.empty
|
||||
else
|
||||
HttpRoutes.of {
|
||||
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
|
||||
offset
|
||||
) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
|
||||
val userQuery =
|
||||
ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse(""))
|
||||
|
||||
Timestamp
|
||||
.current[F]
|
||||
.map(_.toUtcDate)
|
||||
.flatMap(search(userQuery, _))
|
||||
|
||||
case req @ POST -> Root / "search" =>
|
||||
for {
|
||||
userQuery <- req.as[ItemQuery]
|
||||
today <- Timestamp.current[F]
|
||||
resp <- search(userQuery, today.toUtcDate)
|
||||
} yield resp
|
||||
|
||||
case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
|
||||
val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse(""))
|
||||
Timestamp
|
||||
.current[F]
|
||||
.map(_.toUtcDate)
|
||||
.flatMap(searchStats(userQuery, _))
|
||||
|
||||
case req @ POST -> Root / "searchStats" =>
|
||||
for {
|
||||
userQuery <- req.as[ItemQuery]
|
||||
today <- Timestamp.current[F].map(_.toUtcDate)
|
||||
resp <- searchStats(userQuery, today)
|
||||
} yield resp
|
||||
}
|
||||
|
||||
def searchStats(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
|
||||
val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
|
||||
parsedQuery(userQuery, mode)
|
||||
.fold(
|
||||
identity,
|
||||
res =>
|
||||
for {
|
||||
summary <- backend.search.searchSummary(today)(res.q, res.ftq)
|
||||
resp <- Ok(Conversions.mkSearchStats(summary))
|
||||
} yield resp
|
||||
)
|
||||
}
|
||||
|
||||
def search(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
|
||||
val details = userQuery.withDetails.getOrElse(false)
|
||||
val batch =
|
||||
Batch(userQuery.offset.getOrElse(0), userQuery.limit.getOrElse(cfg.maxItemPageSize))
|
||||
.restrictLimitTo(cfg.maxItemPageSize)
|
||||
val limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
|
||||
val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
|
||||
|
||||
parsedQuery(userQuery, mode)
|
||||
.fold(
|
||||
identity,
|
||||
res =>
|
||||
for {
|
||||
items <- backend.search
|
||||
.searchSelect(details, cfg.maxNoteLength, today, batch)(
|
||||
res.q,
|
||||
res.ftq
|
||||
)
|
||||
|
||||
// order is always by date unless q is empty and ftq is not
|
||||
// TODO this is not obvious from the types and an impl detail.
|
||||
ftsOrder = res.q.cond.isEmpty && res.ftq.isDefined
|
||||
|
||||
resp <- Ok(convert(items, batch, limitCapped, ftsOrder))
|
||||
} yield resp
|
||||
)
|
||||
}
|
||||
|
||||
def parsedQuery(
|
||||
userQuery: ItemQuery,
|
||||
mode: SearchMode
|
||||
): Either[F[Response[F]], QueryParseResult.Success] =
|
||||
backend.search.parseQueryString(authToken.account, mode, userQuery.query) match {
|
||||
case s: QueryParseResult.Success =>
|
||||
Right(s)
|
||||
|
||||
case QueryParseResult.ParseFailed(err) =>
|
||||
Left(BadRequest(BasicResult(false, s"Invalid query: $err")))
|
||||
|
||||
case QueryParseResult.FulltextMismatch(FulltextExtract.Result.TooMany) =>
|
||||
Left(
|
||||
BadRequest(
|
||||
BasicResult(false, "Only one fulltext search expression is allowed.")
|
||||
)
|
||||
)
|
||||
case QueryParseResult.FulltextMismatch(
|
||||
FulltextExtract.Result.UnsupportedPosition
|
||||
) =>
|
||||
Left(
|
||||
BadRequest(
|
||||
BasicResult(
|
||||
false,
|
||||
"A fulltext search may only appear in the root and expression."
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def convert(
|
||||
items: Vector[ListItemWithTags],
|
||||
batch: Batch,
|
||||
capped: Boolean,
|
||||
ftsOrder: Boolean
|
||||
): ItemLightList =
|
||||
if (ftsOrder)
|
||||
ItemLightList(
|
||||
List(ItemLightGroup("Results", items.map(convertItem).toList)),
|
||||
batch.limit,
|
||||
batch.offset,
|
||||
capped
|
||||
)
|
||||
else {
|
||||
val groups = items.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
|
||||
|
||||
def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup =
|
||||
ItemLightGroup(g._1, g._2.map(convertItem).toList)
|
||||
|
||||
val gs =
|
||||
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
|
||||
|
||||
ItemLightList(gs, batch.limit, batch.offset, capped)
|
||||
}
|
||||
|
||||
def convertItem(item: ListItemWithTags): ItemLight =
|
||||
ItemLight(
|
||||
id = item.item.id,
|
||||
name = item.item.name,
|
||||
state = item.item.state,
|
||||
date = item.item.date,
|
||||
dueDate = item.item.dueDate,
|
||||
source = item.item.source,
|
||||
direction = item.item.direction.name.some,
|
||||
corrOrg = item.item.corrOrg.map(Conversions.mkIdName),
|
||||
corrPerson = item.item.corrPerson.map(Conversions.mkIdName),
|
||||
concPerson = item.item.concPerson.map(Conversions.mkIdName),
|
||||
concEquipment = item.item.concEquip.map(Conversions.mkIdName),
|
||||
folder = item.item.folder.map(Conversions.mkIdName),
|
||||
attachments = item.attachments.map(Conversions.mkAttachmentLight),
|
||||
tags = item.tags.map(Conversions.mkTag),
|
||||
customfields = item.customfields.map(Conversions.mkItemFieldValue),
|
||||
relatedItems = item.relatedItems,
|
||||
notes = item.item.notes,
|
||||
highlighting = item.item.decodeContext match {
|
||||
case Some(Right(hlctx)) =>
|
||||
hlctx.map(c => HighlightEntry(c.name, c.context))
|
||||
case Some(Left(err)) =>
|
||||
logger.asUnsafe.error(
|
||||
s"Internal error: cannot decode highlight context '${item.item.context}': $err"
|
||||
)
|
||||
Nil
|
||||
case None =>
|
||||
Nil
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@annotation.nowarn
|
||||
object ItemSearchPart {
|
||||
|
||||
def apply[F[_]: Async](
|
||||
backend: BackendApp[F],
|
||||
cfg: Config,
|
||||
authToken: AuthToken
|
||||
): HttpRoutes[F] =
|
||||
if (cfg.featureSearch2) routes(backend, cfg, authToken)
|
||||
else HttpRoutes.empty
|
||||
|
||||
def routes[F[_]: Async](
|
||||
backend: BackendApp[F],
|
||||
cfg: Config,
|
||||
authToken: AuthToken
|
||||
): HttpRoutes[F] = {
|
||||
val logger = docspell.logging.getLogger[F]
|
||||
val dsl = new Http4sDsl[F] {}
|
||||
import dsl._
|
||||
|
||||
HttpRoutes.of {
|
||||
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
|
||||
offset
|
||||
) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
|
||||
???
|
||||
|
||||
case req @ POST -> Root / "search" =>
|
||||
???
|
||||
|
||||
case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
|
||||
???
|
||||
|
||||
case req @ POST -> Root / "searchStats" =>
|
||||
???
|
||||
}
|
||||
}
|
||||
token: AuthToken
|
||||
): ItemSearchPart[F] =
|
||||
new ItemSearchPart[F](backend, cfg, token)
|
||||
}
|
||||
|
||||
@ -23,7 +23,7 @@ trait FtsSupport {
|
||||
val tt = cteTable(ftst)
|
||||
select
|
||||
.appendCte(ftst.distinctCteSimple(tt.tableName))
|
||||
.changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
|
||||
.changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
|
||||
case None =>
|
||||
select
|
||||
}
|
||||
@ -37,7 +37,19 @@ trait FtsSupport {
|
||||
val tt = cteTable(ftst)
|
||||
select
|
||||
.appendCte(ftst.distinctCte(tt.tableName))
|
||||
.changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
|
||||
.changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
|
||||
case None =>
|
||||
select
|
||||
}
|
||||
|
||||
def ftsCondition(
|
||||
itemTable: RItem.Table,
|
||||
ftsTable: Option[TempFtsTable.Table]
|
||||
): Select =
|
||||
ftsTable match {
|
||||
case Some(ftst) =>
|
||||
val ftsIds = Select(ftst.id.s, from(ftst)).distinct
|
||||
select.changeWhere(c => c && itemTable.id.in(ftsIds))
|
||||
case None =>
|
||||
select
|
||||
}
|
||||
|
||||
@ -29,7 +29,7 @@ case class ListItem(
|
||||
|
||||
def decodeContext: Option[Either[String, List[ContextEntry]]] =
|
||||
context.map(_.trim).filter(_.nonEmpty).map { str =>
|
||||
// This is a bit… well. The common denominator for the dbms used is string aggregation
|
||||
// This is a bit…. The common denominator for the dbms used is string aggregation
|
||||
// when combining multiple matches. So the `ContextEntry` objects are concatenated and
|
||||
// separated by comma. TemplateFtsTable ensures than the single entries are all json
|
||||
// objects.
|
||||
@ -40,4 +40,7 @@ case class ListItem(
|
||||
.map(_.getMessage)
|
||||
.map(_.flatten)
|
||||
}
|
||||
|
||||
def toWithTags: ListItemWithTags =
|
||||
ListItemWithTags(this, Nil, Nil, Nil, Nil)
|
||||
}
|
||||
|
||||
@ -475,9 +475,9 @@ object QItem extends FtsSupport {
|
||||
|
||||
val base =
|
||||
findItemsBase(q.fix, today, 0, None).unwrap
|
||||
.joinFtsIdOnly(i, ftsTable)
|
||||
.changeFrom(_.prepend(fieldJoin))
|
||||
.changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
|
||||
.ftsCondition(i, ftsTable)
|
||||
.groupBy(GroupBy(cf.all))
|
||||
|
||||
val basicFields = Nel.of(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user