diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 567d4558..365cdd00 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -47,7 +47,6 @@ trait BackendApp[F[_]] { def userTask: OUserTask[F] def folder: OFolder[F] def customFields: OCustomFields[F] - def simpleSearch: OSimpleSearch[F] def clientSettings: OClientSettings[F] def totp: OTotp[F] def share: OShare[F] @@ -99,8 +98,6 @@ object BackendApp { itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs) itemSearchImpl <- OItemSearch(store) fulltextImpl <- OFulltext( - itemSearchImpl, - ftsClient, store, schedulerModule.jobs ) @@ -112,15 +109,15 @@ object BackendApp { ) folderImpl <- OFolder(store) customFieldsImpl <- OCustomFields(store) - simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl) clientSettingsImpl <- OClientSettings(store) + searchImpl <- Resource.pure(OSearch(store, ftsClient)) shareImpl <- Resource.pure( - OShare(store, itemSearchImpl, simpleSearchImpl, javaEmil) + OShare(store, itemSearchImpl, searchImpl, javaEmil) ) notifyImpl <- ONotification(store, notificationMod) bookmarksImpl <- OQueryBookmarks(store) fileRepoImpl <- OFileRepository(store, schedulerModule.jobs) - itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl)) + itemLinkImpl <- Resource.pure(OItemLink(store, searchImpl)) downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs)) attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs)) addonsImpl <- Resource.pure( @@ -132,7 +129,6 @@ object BackendApp { joexImpl ) ) - searchImpl <- Resource.pure(OSearch(store, ftsClient)) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl @@ -153,7 +149,6 @@ object BackendApp { val userTask = userTaskImpl val folder = folderImpl val customFields = customFieldsImpl - val simpleSearch = simpleSearchImpl val clientSettings = clientSettingsImpl val totp = totpImpl val share = shareImpl diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 80e500a6..89b3fa82 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -6,46 +6,17 @@ package docspell.backend.ops -import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ -import fs2.Stream import docspell.backend.JobFactory -import docspell.backend.ops.OItemSearch._ import docspell.common._ -import docspell.ftsclient._ -import docspell.query.ItemQuery._ -import docspell.query.ItemQueryDsl._ import docspell.scheduler.JobStore -import docspell.store.queries.{QFolder, QItem, SelectedItem} +import docspell.store.Store import docspell.store.records.RJob -import docspell.store.{Store, qb} trait OFulltext[F[_]] { - def findItems(maxNoteLen: Int)( - q: Query, - fts: OFulltext.FtsInput, - batch: qb.Batch - ): F[Vector[OFulltext.FtsItem]] - - /** Same as `findItems` but does more queries per item to find all tags. */ - def findItemsWithTags(maxNoteLen: Int)( - q: Query, - fts: OFulltext.FtsInput, - batch: qb.Batch - ): F[Vector[OFulltext.FtsItemWithTags]] - - def findIndexOnly(maxNoteLen: Int)( - fts: OFulltext.FtsInput, - account: AccountId, - batch: qb.Batch - ): F[Vector[OFulltext.FtsItemWithTags]] - - def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary] - def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary] - /** Clears the full-text index completely and launches a task that indexes all data. */ def reindexAll: F[Unit] @@ -56,30 +27,7 @@ trait OFulltext[F[_]] { } object OFulltext { - - case class FtsInput( - query: String, - highlightPre: String = "***", - highlightPost: String = "***" - ) - - case class FtsDataItem( - score: Double, - matchData: FtsResult.MatchData, - context: List[String] - ) - case class FtsData( - maxScore: Double, - count: Int, - qtime: Duration, - items: List[FtsDataItem] - ) - case class FtsItem(item: ListItem, ftsData: FtsData) - case class FtsItemWithTags(item: ListItemWithTags, ftsData: FtsData) - def apply[F[_]: Async]( - itemSearch: OItemSearch[F], - fts: FtsClient[F], store: Store[F], jobStore: JobStore[F] ): Resource[F, OFulltext[F]] = @@ -103,232 +51,5 @@ object OFulltext { if (exist.isDefined) ().pure[F] else jobStore.insertIfNew(job.encode) } yield () - - def findIndexOnly(maxNoteLen: Int)( - ftsQ: OFulltext.FtsInput, - account: AccountId, - batch: qb.Batch - ): F[Vector[OFulltext.FtsItemWithTags]] = { - val fq = FtsQuery( - ftsQ.query, - account.collective, - Set.empty, - Set.empty, - batch.limit, - batch.offset, - FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost) - ) - for { - _ <- logger.trace(s"Find index only: ${ftsQ.query}/$batch") - folders <- store.transact(QFolder.getMemberFolders(account)) - ftsR <- fts.search(fq.withFolders(folders)) - ftsItems = ftsR.results.groupBy(_.itemId) - select = - ftsItems.values - .map(_.minBy(-_.score)) - .map(r => SelectedItem(r.itemId, r.score)) - .toSet - now <- Timestamp.current[F] - itemsWithTags <- - store - .transact( - QItem.findItemsWithTags( - account.collective, - QItem.findSelectedItems( - Query.all(account), - now.toUtcDate, - maxNoteLen, - select - ) - ) - ) - .take(batch.limit.toLong) - .compile - .toVector - res = - itemsWithTags - .collect(convertFtsData(ftsR, ftsItems)) - .map { case (li, fd) => FtsItemWithTags(li, fd) } - } yield res - } - - def findIndexOnlySummary( - account: AccountId, - ftsQ: OFulltext.FtsInput - ): F[SearchSummary] = { - val fq = FtsQuery( - ftsQ.query, - account.collective, - Set.empty, - Set.empty, - 500, - 0, - FtsQuery.HighlightSetting.default - ) - - for { - folder <- store.transact(QFolder.getMemberFolders(account)) - now <- Timestamp.current[F] - itemIds <- fts - .searchAll(fq.withFolders(folder)) - .flatMap(r => Stream.emits(r.results.map(_.itemId))) - .compile - .to(Set) - itemIdsQuery = NonEmptyList - .fromList(itemIds.toList) - .map(ids => Attr.ItemId.in(ids.map(_.id))) - .getOrElse(Attr.ItemId.notExists) - q = Query - .all(account) - .withFix(_.copy(query = itemIdsQuery.some)) - res <- store.transact(QItem.searchStats(now.toUtcDate, None)(q)) - } yield res - } - - def findItems( - maxNoteLen: Int - )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] = - findItemsFts( - q, - ftsQ, - batch.first, - itemSearch.findItems(maxNoteLen), - convertFtsData[ListItem] - ) - .drop(batch.offset.toLong) - .take(batch.limit.toLong) - .map { case (li, fd) => FtsItem(li, fd) } - .compile - .toVector - - def findItemsWithTags(maxNoteLen: Int)( - q: Query, - ftsQ: FtsInput, - batch: qb.Batch - ): F[Vector[FtsItemWithTags]] = - findItemsFts( - q, - ftsQ, - batch.first, - itemSearch.findItemsWithTags(maxNoteLen), - convertFtsData[ListItemWithTags] - ) - .drop(batch.offset.toLong) - .take(batch.limit.toLong) - .map { case (li, fd) => FtsItemWithTags(li, fd) } - .compile - .toVector - - def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] = - for { - search <- itemSearch.findItems(0)(q, Batch.all) - fq = FtsQuery( - ftsQ.query, - q.fix.account.collective, - search.map(_.id).toSet, - Set.empty, - 500, - 0, - FtsQuery.HighlightSetting.default - ) - items <- fts - .searchAll(fq) - .flatMap(r => Stream.emits(r.results.map(_.itemId))) - .compile - .to(Set) - itemIdsQuery = NonEmptyList - .fromList(items.toList) - .map(ids => Attr.ItemId.in(ids.map(_.id))) - .getOrElse(Attr.ItemId.notExists) - qnext = q.withFix(_.copy(query = itemIdsQuery.some)) - now <- Timestamp.current[F] - res <- store.transact(QItem.searchStats(now.toUtcDate, None)(qnext)) - } yield res - - // Helper - - private def findItemsFts[A: ItemId, B]( - q: Query, - ftsQ: FtsInput, - batch: qb.Batch, - search: (Query, qb.Batch) => F[Vector[A]], - convert: ( - FtsResult, - Map[Ident, List[FtsResult.ItemMatch]] - ) => PartialFunction[A, (A, FtsData)] - ): Stream[F, (A, FtsData)] = - findItemsFts0(q, ftsQ, batch, search, convert) - .takeThrough(_._1 >= batch.limit) - .flatMap(x => Stream.emits(x._2)) - - private def findItemsFts0[A: ItemId, B]( - q: Query, - ftsQ: FtsInput, - batch: qb.Batch, - search: (Query, qb.Batch) => F[Vector[A]], - convert: ( - FtsResult, - Map[Ident, List[FtsResult.ItemMatch]] - ) => PartialFunction[A, (A, FtsData)] - ): Stream[F, (Int, Vector[(A, FtsData)])] = { - val sqlResult = search(q, batch) - val fq = FtsQuery( - ftsQ.query, - q.fix.account.collective, - Set.empty, - Set.empty, - 0, - 0, - FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost) - ) - - val qres = - for { - items <- sqlResult - ids = items.map(a => ItemId[A].itemId(a)) - idsNel = NonEmptyList.fromFoldable(ids) - // must find all index results involving the items. - // Currently there is one result per item + one result per - // attachment - limit <- idsNel - .map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds))) - .getOrElse(0.pure[F]) - ftsQ = fq.copy(items = ids.toSet, limit = limit) - ftsR <- fts.search(ftsQ) - ftsItems = ftsR.results.groupBy(_.itemId) - res = items.collect(convert(ftsR, ftsItems)) - } yield (items.size, res) - - Stream.eval(qres) ++ findItemsFts0(q, ftsQ, batch.next, search, convert) - } - - private def convertFtsData[A: ItemId]( - ftr: FtsResult, - ftrItems: Map[Ident, List[FtsResult.ItemMatch]] - ): PartialFunction[A, (A, FtsData)] = { - case a if ftrItems.contains(ItemId[A].itemId(a)) => - val ftsDataItems = ftrItems - .getOrElse(ItemId[A].itemId(a), Nil) - .map(im => - FtsDataItem(im.score, im.data, ftr.highlight.getOrElse(im.id, Nil)) - ) - (a, FtsData(ftr.maxScore, ftr.count, ftr.qtime, ftsDataItems)) - } }) - - trait ItemId[A] { - def itemId(a: A): Ident - } - object ItemId { - def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev - - def from[A](f: A => Ident): ItemId[A] = - (a: A) => f(a) - - implicit val listItemId: ItemId[ListItem] = - ItemId.from(_.id) - - implicit val listItemWithTagsId: ItemId[ListItemWithTags] = - ItemId.from(_.item.id) - } } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala index 16077b10..457c2b42 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala @@ -11,11 +11,12 @@ import cats.effect._ import cats.implicits._ import docspell.backend.ops.OItemLink.LinkResult +import docspell.backend.ops.search.OSearch import docspell.common.{AccountId, Ident} import docspell.query.ItemQuery import docspell.query.ItemQueryDsl._ import docspell.store.qb.Batch -import docspell.store.queries.Query +import docspell.store.queries.{ListItemWithTags, Query} import docspell.store.records.RItemLink import docspell.store.{AddResult, Store} @@ -29,7 +30,7 @@ trait OItemLink[F[_]] { account: AccountId, item: Ident, batch: Batch - ): F[Vector[OItemSearch.ListItemWithTags]] + ): F[Vector[ListItemWithTags]] } object OItemLink { @@ -44,13 +45,13 @@ object OItemLink { def linkTargetItemError: LinkResult = LinkTargetItemError } - def apply[F[_]: Sync](store: Store[F], search: OItemSearch[F]): OItemLink[F] = + def apply[F[_]: Sync](store: Store[F], search: OSearch[F]): OItemLink[F] = new OItemLink[F] { def getRelated( accountId: AccountId, item: Ident, batch: Batch - ): F[Vector[OItemSearch.ListItemWithTags]] = + ): F[Vector[ListItemWithTags]] = store .transact(RItemLink.findLinked(accountId.collective, item)) .map(ids => NonEmptyList.fromList(ids.toList)) @@ -62,10 +63,10 @@ object OItemLink { .Fix(accountId, Some(ItemQuery.Expr.ValidItemStates), None), Query.QueryExpr(expr) ) - search.findItemsWithTags(0)(query, batch) + search.searchWithDetails(0, None, batch)(query, None) case None => - Vector.empty[OItemSearch.ListItemWithTags].pure[F] + Vector.empty[ListItemWithTags].pure[F] } def addAll(cid: Ident, target: Ident, related: NonEmptyList[Ident]): F[LinkResult] = diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index 1ce4e166..b56b850a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -25,15 +25,6 @@ trait OItemSearch[F[_]] { def findDeleted(collective: Ident, maxUpdate: Timestamp, limit: Int): F[Vector[RItem]] - def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] - - /** Same as `findItems` but does more queries per item to find all tags. */ - def findItemsWithTags( - maxNoteLen: Int - )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] - - def findItemsSummary(q: Query): F[SearchSummary] - def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] def findAttachmentSource( @@ -63,9 +54,6 @@ trait OItemSearch[F[_]] { object OItemSearch { - type SearchSummary = queries.SearchSummary - val SearchSummary = queries.SearchSummary - type CustomValue = queries.CustomValue val CustomValue = queries.CustomValue @@ -75,12 +63,6 @@ object OItemSearch { type Batch = qb.Batch val Batch = docspell.store.qb.Batch - type ListItem = queries.ListItem - val ListItem = queries.ListItem - - type ListItemWithTags = queries.ListItemWithTags - val ListItemWithTags = queries.ListItemWithTags - type ItemFieldValue = queries.ItemFieldValue val ItemFieldValue = queries.ItemFieldValue @@ -136,19 +118,6 @@ object OItemSearch { store .transact(QItem.findItem(id, collective)) - def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] = - Timestamp - .current[F] - .map(_.toUtcDate) - .flatMap { today => - store - .transact( - QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong) - ) - .compile - .toVector - } - def findDeleted( collective: Ident, maxUpdate: Timestamp, @@ -160,28 +129,6 @@ object OItemSearch { .compile .toVector - def findItemsWithTags( - maxNoteLen: Int - )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = - for { - now <- Timestamp.current[F] - search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch) - res <- store - .transact( - QItem - .findItemsWithTags(q.fix.account.collective, search) - .take(batch.limit.toLong) - ) - .compile - .toVector - } yield res - - def findItemsSummary(q: Query): F[SearchSummary] = - Timestamp - .current[F] - .map(_.toUtcDate) - .flatMap(today => store.transact(QItem.searchStats(today, None)(q))) - def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store .transact(RAttachment.findByIdAndCollective(id, collective)) @@ -298,6 +245,5 @@ object OItemSearch { coll <- OptionT(RSource.findCollective(sourceId)) items <- OptionT.liftF(QItem.findByChecksum(checksum, coll, Set.empty)) } yield items).value) - }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OShare.scala b/modules/backend/src/main/scala/docspell/backend/ops/OShare.scala index 1f393451..0f4472f3 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OShare.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OShare.scala @@ -14,13 +14,12 @@ import docspell.backend.PasswordCrypt import docspell.backend.auth.ShareToken import docspell.backend.ops.OItemSearch._ import docspell.backend.ops.OShare._ -import docspell.backend.ops.OSimpleSearch.StringSearchResult +import docspell.backend.ops.search.{OSearch, QueryParseResult} import docspell.common._ import docspell.query.ItemQuery.Expr import docspell.query.ItemQuery.Expr.AttachId import docspell.query.{FulltextExtract, ItemQuery} import docspell.store.Store -import docspell.store.queries.SearchSummary import docspell.store.records._ import emil._ @@ -67,9 +66,10 @@ trait OShare[F[_]] { def findItem(itemId: Ident, shareId: Ident): OptionT[F, ItemData] - def searchSummary( - settings: OSimpleSearch.StatsSettings - )(shareId: Ident, q: ItemQueryString): OptionT[F, StringSearchResult[SearchSummary]] + /** Parses a query and amends the result with the stored query of the share. The result + * can be used with [[OSearch]] to search for items. + */ + def parseQuery(share: ShareQuery, qs: String): QueryParseResult def sendMail(account: AccountId, connection: Ident, mail: ShareMail): F[SendResult] } @@ -148,7 +148,7 @@ object OShare { def apply[F[_]: Async]( store: Store[F], itemSearch: OItemSearch[F], - simpleSearch: OSimpleSearch[F], + search: OSearch[F], emil: Emil[F] ): OShare[F] = new OShare[F] { @@ -325,8 +325,8 @@ object OShare { Query.QueryExpr(idExpr) ) OptionT( - itemSearch - .findItems(0)(checkQuery, Batch.limit(1)) + search + .search(0, None, Batch.limit(1))(checkQuery, None) .map(_.headOption.map(_ => ())) ).flatTapNone( logger.info( @@ -335,22 +335,11 @@ object OShare { ) } - def searchSummary( - settings: OSimpleSearch.StatsSettings - )( - shareId: Ident, - q: ItemQueryString - ): OptionT[F, StringSearchResult[SearchSummary]] = - findShareQuery(shareId) - .semiflatMap { share => - val fix = Query.Fix(share.account, Some(share.query.expr), None) - simpleSearch - .searchSummaryByString(settings)(fix, q) - .map { - case StringSearchResult.Success(summary) => - StringSearchResult.Success(summary.onlyExisting) - case other => other - } + def parseQuery(share: ShareQuery, qs: String): QueryParseResult = + search + .parseQueryString(share.account, SearchMode.Normal, qs) + .map { case QueryParseResult.Success(q, ftq) => + QueryParseResult.Success(q.withFix(_.andQuery(share.query.expr)), ftq) } def sendMail( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala deleted file mode 100644 index 60502813..00000000 --- a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.backend.ops - -import cats.Applicative -import cats.effect.Sync -import cats.implicits._ - -import docspell.backend.ops.OSimpleSearch._ -import docspell.common._ -import docspell.query._ -import docspell.store.qb.Batch -import docspell.store.queries.Query -import docspell.store.queries.SearchSummary - -import org.log4s.getLogger - -/** A "porcelain" api on top of OFulltext and OItemSearch. This takes care of restricting - * the items to a subset, e.g. only items that have a "valid" state. - */ -trait OSimpleSearch[F[_]] { - - /** Search for items using the given query and optional fulltext search. - * - * When using fulltext search only (the query is empty), only the index is searched. It - * is assumed that the index doesn't contain "invalid" items. When using a query, then - * a condition to select only valid items is added to it. - */ - def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items] - - /** Using the same arguments as in `search`, this returns a summary and not the results. - */ - def searchSummary( - settings: StatsSettings - )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] - - /** Calls `search` by parsing the given query string into a query that is then amended - * wtih the given `fix` query. - */ - final def searchByString( - settings: Settings - )(fix: Query.Fix, q: ItemQueryString)(implicit - F: Applicative[F] - ): F[StringSearchResult[Items]] = - OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts)) - - /** Same as `searchByString` but returning a summary instead of the results. */ - final def searchSummaryByString( - settings: StatsSettings - )(fix: Query.Fix, q: ItemQueryString)(implicit - F: Applicative[F] - ): F[StringSearchResult[SearchSummary]] = - OSimpleSearch.applySearch[F, SearchSummary](fix, q)((iq, fts) => - searchSummary(settings)(iq, fts) - ) -} - -object OSimpleSearch { - private[this] val logger = getLogger - - sealed trait StringSearchResult[+A] - object StringSearchResult { - case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing] - def parseFailed[A](error: ParseFailure): StringSearchResult[A] = - ParseFailed(error) - - case class FulltextMismatch(error: FulltextExtract.FailureResult) - extends StringSearchResult[Nothing] - def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] = - FulltextMismatch(error) - - case class Success[A](value: A) extends StringSearchResult[A] - } - - final case class Settings( - batch: Batch, - useFTS: Boolean, - resolveDetails: Boolean, - maxNoteLen: Int, - searchMode: SearchMode - ) - final case class StatsSettings( - useFTS: Boolean, - searchMode: SearchMode - ) - - sealed trait Items { - def fold[A]( - f1: Items.FtsItems => A, - f2: Items.FtsItemsFull => A, - f3: Vector[OItemSearch.ListItem] => A, - f4: Vector[OItemSearch.ListItemWithTags] => A - ): A - - } - object Items { - def ftsItems(indexOnly: Boolean)(items: Vector[OFulltext.FtsItem]): Items = - FtsItems(items, indexOnly) - - case class FtsItems(items: Vector[OFulltext.FtsItem], indexOnly: Boolean) - extends Items { - def fold[A]( - f1: FtsItems => A, - f2: FtsItemsFull => A, - f3: Vector[OItemSearch.ListItem] => A, - f4: Vector[OItemSearch.ListItemWithTags] => A - ): A = f1(this) - - } - - def ftsItemsFull(indexOnly: Boolean)( - items: Vector[OFulltext.FtsItemWithTags] - ): Items = - FtsItemsFull(items, indexOnly) - - case class FtsItemsFull(items: Vector[OFulltext.FtsItemWithTags], indexOnly: Boolean) - extends Items { - def fold[A]( - f1: FtsItems => A, - f2: FtsItemsFull => A, - f3: Vector[OItemSearch.ListItem] => A, - f4: Vector[OItemSearch.ListItemWithTags] => A - ): A = f2(this) - } - - def itemsPlain(items: Vector[OItemSearch.ListItem]): Items = - ItemsPlain(items) - - case class ItemsPlain(items: Vector[OItemSearch.ListItem]) extends Items { - def fold[A]( - f1: FtsItems => A, - f2: FtsItemsFull => A, - f3: Vector[OItemSearch.ListItem] => A, - f4: Vector[OItemSearch.ListItemWithTags] => A - ): A = f3(items) - } - - def itemsFull(items: Vector[OItemSearch.ListItemWithTags]): Items = - ItemsFull(items) - - case class ItemsFull(items: Vector[OItemSearch.ListItemWithTags]) extends Items { - def fold[A]( - f1: FtsItems => A, - f2: FtsItemsFull => A, - f3: Vector[OItemSearch.ListItem] => A, - f4: Vector[OItemSearch.ListItemWithTags] => A - ): A = f4(items) - } - - } - - def apply[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]): OSimpleSearch[F] = - new Impl(fts, is) - - /** Parses the query and calls `run` with the result, which searches items. */ - private def applySearch[F[_]: Applicative, A](fix: Query.Fix, q: ItemQueryString)( - run: (Query, Option[String]) => F[A] - ): F[StringSearchResult[A]] = { - val parsed: Either[StringSearchResult[A], Option[ItemQuery]] = - if (q.isEmpty) Right(None) - else - ItemQueryParser - .parse(q.query) - .leftMap(StringSearchResult.parseFailed) - .map(_.some) - - def makeQuery(itemQuery: Option[ItemQuery]): F[StringSearchResult[A]] = - runQuery[F, A](itemQuery) { - case Some(s) => - run(Query(fix, Query.QueryExpr(s.getExprPart)), s.getFulltextPart) - case None => - run(Query(fix), None) - } - - parsed match { - case Right(iq) => - makeQuery(iq) - case Left(err) => - err.pure[F] - } - } - - /** Calls `run` with one of the success results when extracting the fulltext search node - * from the query. - */ - private def runQuery[F[_]: Applicative, A]( - itemQuery: Option[ItemQuery] - )(run: Option[FulltextExtract.SuccessResult] => F[A]): F[StringSearchResult[A]] = - itemQuery match { - case Some(iq) => - iq.findFulltext match { - case s: FulltextExtract.SuccessResult => - run(Some(s)).map(StringSearchResult.Success.apply) - case other: FulltextExtract.FailureResult => - StringSearchResult.fulltextMismatch[A](other).pure[F] - } - case None => - run(None).map(StringSearchResult.Success.apply) - } - - final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]) - extends OSimpleSearch[F] { - - /** Implements searching like this: it exploits the fact that teh fulltext index only - * contains valid items. When searching via sql the query expression selecting only - * valid items is added here. - */ - def search( - settings: Settings - )(q: Query, fulltextQuery: Option[String]): F[Items] = { - // 1. fulltext only if fulltextQuery.isDefined && q.isEmpty && useFTS - // 2. sql+fulltext if fulltextQuery.isDefined && q.nonEmpty && useFTS - // 3. sql-only else (if fulltextQuery.isEmpty || !useFTS) - val validItemQuery = - settings.searchMode match { - case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed)) - case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates)) - case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed)) - } - fulltextQuery match { - case Some(ftq) if settings.useFTS => - if (q.isEmpty) { - logger.debug(s"Using index only search: $fulltextQuery") - if (settings.searchMode == SearchMode.Trashed) - Items.ftsItemsFull(true)(Vector.empty).pure[F] - else - fts - .findIndexOnly(settings.maxNoteLen)( - OFulltext.FtsInput(ftq), - q.fix.account, - settings.batch - ) - .map(Items.ftsItemsFull(true)) - } else if (settings.resolveDetails) { - logger.debug( - s"Using index+sql search with tags: $validItemQuery / $fulltextQuery" - ) - fts - .findItemsWithTags(settings.maxNoteLen)( - validItemQuery, - OFulltext.FtsInput(ftq), - settings.batch - ) - .map(Items.ftsItemsFull(false)) - } else { - logger.debug( - s"Using index+sql search no tags: $validItemQuery / $fulltextQuery" - ) - fts - .findItems(settings.maxNoteLen)( - validItemQuery, - OFulltext.FtsInput(ftq), - settings.batch - ) - .map(Items.ftsItems(false)) - } - case _ => - if (settings.resolveDetails) { - logger.debug( - s"Using sql only search with tags: $validItemQuery / $fulltextQuery" - ) - is.findItemsWithTags(settings.maxNoteLen)(validItemQuery, settings.batch) - .map(Items.itemsFull) - } else { - logger.debug( - s"Using sql only search no tags: $validItemQuery / $fulltextQuery" - ) - is.findItems(settings.maxNoteLen)(validItemQuery, settings.batch) - .map(Items.itemsPlain) - } - } - } - - def searchSummary( - settings: StatsSettings - )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = { - val validItemQuery = - settings.searchMode match { - case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed)) - case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates)) - case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed)) - } - fulltextQuery match { - case Some(ftq) if settings.useFTS => - if (q.isEmpty) - fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq)) - else - fts - .findItemsSummary(validItemQuery, OFulltext.FtsInput(ftq)) - - case _ => - is.findItemsSummary(validItemQuery) - } - } - } -} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala index 17e7412e..483f8fc7 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala @@ -8,13 +8,13 @@ package docspell.backend.ops.search import java.time.LocalDate +import cats.data.OptionT import cats.effect._ import cats.syntax.all._ import cats.{Functor, ~>} import fs2.Stream -import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary} -import docspell.common.{AccountId, Duration, SearchMode} +import docspell.common._ import docspell.ftsclient.{FtsClient, FtsQuery} import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser} import docspell.store.Store @@ -35,7 +35,7 @@ trait OSearch[F[_]] { * from fulltext search. Any "fulltext search" query node is discarded. It is assumed * that the fulltext search node has been extracted into the argument. */ - def search(maxNoteLen: Int, today: LocalDate, batch: Batch)( + def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)( q: Query, fulltextQuery: Option[String] ): F[Vector[ListItem]] @@ -45,7 +45,7 @@ trait OSearch[F[_]] { */ def searchWithDetails( maxNoteLen: Int, - today: LocalDate, + today: Option[LocalDate], batch: Batch )( q: Query, @@ -58,7 +58,7 @@ trait OSearch[F[_]] { final def searchSelect( withDetails: Boolean, maxNoteLen: Int, - today: LocalDate, + today: Option[LocalDate], batch: Batch )( q: Query, @@ -69,12 +69,14 @@ trait OSearch[F[_]] { /** Run multiple database calls with the give query to collect a summary. */ def searchSummary( - today: LocalDate + today: Option[LocalDate] )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] /** Parses a query string and creates a `Query` object, to be used with the other - * methods. The query object contains the parsed query amended with more conditions to - * restrict to valid items only (as specified with `mode`). + * methods. The query object contains the parsed query amended with more conditions, + * for example to restrict to valid items only (as specified with `mode`). An empty + * query string is allowed and returns a query containing only the restrictions in the + * `q.fix` part. */ def parseQueryString( accountId: AccountId, @@ -139,7 +141,7 @@ object OSearch { } } - def search(maxNoteLen: Int, today: LocalDate, batch: Batch)( + def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)( q: Query, fulltextQuery: Option[String] ): F[Vector[ListItem]] = @@ -148,6 +150,9 @@ object OSearch { for { timed <- Duration.stopTime[F] ftq <- createFtsQuery(q.fix.account, ftq) + date <- OptionT + .fromOption(today) + .getOrElseF(Timestamp.current[F].map(_.toUtcDate)) results <- WeakAsync.liftK[F, ConnectionIO].use { nat => val tempTable = temporaryFtsTable(ftq, nat) @@ -156,7 +161,7 @@ object OSearch { Stream .eval(tempTable) .flatMap(tt => - QItem.queryItems(q, today, maxNoteLen, batch, tt.some) + QItem.queryItems(q, date, maxNoteLen, batch, tt.some) ) ) .compile @@ -169,19 +174,21 @@ object OSearch { case None => for { timed <- Duration.stopTime[F] + date <- OptionT + .fromOption(today) + .getOrElseF(Timestamp.current[F].map(_.toUtcDate)) results <- store - .transact(QItem.queryItems(q, today, maxNoteLen, batch, None)) + .transact(QItem.queryItems(q, date, maxNoteLen, batch, None)) .compile .toVector duration <- timed _ <- logger.debug(s"Simple search sql in: ${duration.formatExact}") } yield results - } def searchWithDetails( maxNoteLen: Int, - today: LocalDate, + today: Option[LocalDate], batch: Batch )( q: Query, @@ -201,22 +208,28 @@ object OSearch { } yield resolved def searchSummary( - today: LocalDate + today: Option[LocalDate] )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = fulltextQuery match { case Some(ftq) => for { ftq <- createFtsQuery(q.fix.account, ftq) + date <- OptionT + .fromOption(today) + .getOrElseF(Timestamp.current[F].map(_.toUtcDate)) results <- WeakAsync.liftK[F, ConnectionIO].use { nat => val tempTable = temporaryFtsTable(ftq, nat) store.transact( - tempTable.flatMap(tt => QItem.searchStats(today, tt.some)(q)) + tempTable.flatMap(tt => QItem.searchStats(date, tt.some)(q)) ) } } yield results case None => - store.transact(QItem.searchStats(today, None)(q)) + OptionT + .fromOption(today) + .getOrElseF(Timestamp.current[F].map(_.toUtcDate)) + .flatMap(date => store.transact(QItem.searchStats(date, None)(q))) } private def createFtsQuery( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala index 1faf2275..4442c098 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala @@ -15,6 +15,8 @@ sealed trait QueryParseResult { def get: Option[(Query, Option[String])] def isSuccess: Boolean = get.isDefined def isFailure: Boolean = !isSuccess + + def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult } object QueryParseResult { @@ -25,15 +27,22 @@ object QueryParseResult { def withFtsEnabled(enabled: Boolean) = if (enabled || ftq.isEmpty) this else copy(ftq = None) + def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult = + f(this) + val get = Some(q -> ftq) } final case class ParseFailed(error: ParseFailure) extends QueryParseResult { val get = None + def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult = + this } final case class FulltextMismatch(error: FulltextExtract.FailureResult) extends QueryParseResult { val get = None + def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult = + this } } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala index f5c99d47..7114be6d 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala @@ -13,6 +13,7 @@ import docspell.backend.BackendCommands import docspell.backend.fulltext.CreateIndex import docspell.backend.joex.AddonOps import docspell.backend.ops._ +import docspell.backend.ops.search.OSearch import docspell.backend.task.DownloadZipArgs import docspell.common._ import docspell.config.FtsType @@ -62,6 +63,7 @@ final class JoexTasks[F[_]: Async]( joex: OJoex[F], jobs: OJob[F], itemSearch: OItemSearch[F], + search: OSearch[F], addons: AddonOps[F] ) { val downloadAll: ODownloadAll[F] = @@ -201,7 +203,7 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( PeriodicQueryTask.taskName, - PeriodicQueryTask[F](store, notification), + PeriodicQueryTask[F](store, search, notification), PeriodicQueryTask.onCancel[F] ) ) @@ -273,6 +275,7 @@ object JoexTasks { createIndex <- CreateIndex.resource(fts, store) itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs) itemSearchOps <- OItemSearch(store) + searchOps <- Resource.pure(OSearch(store, fts)) analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig) regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) updateCheck <- UpdateCheck.resource(httpClient) @@ -306,6 +309,7 @@ object JoexTasks { joex, jobs, itemSearchOps, + searchOps, addons ) diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala index 94db119b..3f2576aa 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala @@ -89,7 +89,7 @@ object PeriodicDueItemsTask { store .transact( QItem - .findItems(q, now.toUtcDate, 0, Batch.limit(limit)) + .queryItems(q, now.toUtcDate, 0, Batch.limit(limit), None) .take(limit.toLong) ) .compile diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala index af1242cd..fbfa127f 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala @@ -7,25 +7,21 @@ package docspell.joex.notify import cats.data.OptionT -import cats.data.{NonEmptyList => Nel} import cats.effect._ import cats.implicits._ import docspell.backend.ops.ONotification +import docspell.backend.ops.search.{OSearch, QueryParseResult} import docspell.common._ import docspell.notification.api.EventContext import docspell.notification.api.NotificationChannel import docspell.notification.api.PeriodicQueryArgs -import docspell.query.ItemQuery -import docspell.query.ItemQuery.Expr -import docspell.query.ItemQuery.Expr.AndExpr -import docspell.query.ItemQueryParser +import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser} import docspell.scheduler.Context import docspell.scheduler.Task import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.ListItem -import docspell.store.queries.{QItem, Query} import docspell.store.records.RQueryBookmark import docspell.store.records.RShare @@ -39,12 +35,13 @@ object PeriodicQueryTask { def apply[F[_]: Sync]( store: Store[F], + search: OSearch[F], notificationOps: ONotification[F] ): Task[F, Args, Unit] = Task { ctx => val limit = 7 Timestamp.current[F].flatMap { now => - withItems(ctx, store, limit, now) { items => + withItems(ctx, store, search, limit, now) { items => withEventContext(ctx, items, limit, now) { eventCtx => withChannel(ctx, notificationOps) { channels => notificationOps.sendMessage(ctx.logger, eventCtx, channels) @@ -62,8 +59,8 @@ object PeriodicQueryTask { private def queryString(q: ItemQuery.Expr) = ItemQueryParser.asString(q) - def withQuery[F[_]: Sync](ctx: Context[F, Args], store: Store[F])( - cont: Query => F[Unit] + def withQuery[F[_]: Sync](ctx: Context[F, Args], store: Store[F], search: OSearch[F])( + cont: QueryParseResult.Success => F[Unit] ): F[Unit] = { def fromBookmark(id: String) = store @@ -84,33 +81,51 @@ object PeriodicQueryTask { def fromBookmarkOrShare(id: String) = OptionT(fromBookmark(id)).orElse(OptionT(fromShare(id))).value - def runQuery(bm: Option[ItemQuery], str: String): F[Unit] = - ItemQueryParser.parse(str) match { - case Right(q) => - val expr = bm.map(b => AndExpr(Nel.of(b.expr, q.expr))).getOrElse(q.expr) - val query = Query - .all(ctx.args.account) - .withFix(_.copy(query = Expr.ValidItemStates.some)) - .withCond(_ => Query.QueryExpr(expr)) + def runQuery(bm: Option[ItemQuery], str: Option[String]): F[Unit] = { + val bmFtsQuery = bm.map(e => FulltextExtract.findFulltext(e.expr)) + val queryStrResult = + str.map(search.parseQueryString(ctx.args.account, SearchMode.Normal, _)) - ctx.logger.debug(s"Running query: ${queryString(expr)}") *> cont(query) + (bmFtsQuery, queryStrResult) match { + case ( + Some(bmr: FulltextExtract.SuccessResult), + Some(QueryParseResult.Success(q, ftq)) + ) => + val nq = bmr.getExprPart.map(q.andCond).getOrElse(q) + val nftq = + (bmr.getFulltextPart |+| Some(" ") |+| ftq).map(_.trim).filter(_.nonEmpty) + val r = QueryParseResult.Success(nq, nftq) + ctx.logger.debug(s"Running query: $r") *> cont(r) - case Left(err) => - ctx.logger.error( - s"Item query is invalid, stopping: ${ctx.args.query.map(_.query)} - ${err.render}" - ) + case (None, Some(r: QueryParseResult.Success)) => + ctx.logger.debug(s"Running query: $r") *> cont(r) + + case (Some(bmr: FulltextExtract.SuccessResult), None) => + search.parseQueryString(ctx.args.account, SearchMode.Normal, "") match { + case QueryParseResult.Success(q, _) => + val nq = bmr.getExprPart.map(q.andCond).getOrElse(q) + ctx.logger.debug(s"Running query: $nq") *> + cont(QueryParseResult.Success(nq, bmr.getFulltextPart)) + + case err => + ctx.logger.error(s"Internal error: $err") + } + + case (failure1, res2) => + ctx.logger.error(s"One or more error reading queries: $failure1 and $res2") } + } (ctx.args.bookmark, ctx.args.query) match { case (Some(bm), Some(qstr)) => ctx.logger.debug(s"Using bookmark $bm and query $qstr") *> - fromBookmarkOrShare(bm).flatMap(bq => runQuery(bq, qstr.query)) + fromBookmarkOrShare(bm).flatMap(bq => runQuery(bq, qstr.query.some)) case (Some(bm), None) => fromBookmarkOrShare(bm).flatMap { case Some(bq) => - val query = Query(Query.Fix(ctx.args.account, Some(bq.expr), None)) - ctx.logger.debug(s"Using bookmark: ${queryString(bq.expr)}") *> cont(query) + ctx.logger.debug(s"Using bookmark: ${queryString(bq.expr)}") *> + runQuery(bq.some, None) case None => ctx.logger.error( @@ -119,7 +134,7 @@ object PeriodicQueryTask { } case (None, Some(qstr)) => - ctx.logger.debug(s"Using query: ${qstr.query}") *> runQuery(None, qstr.query) + ctx.logger.debug(s"Using query: ${qstr.query}") *> runQuery(None, qstr.query.some) case (None, None) => ctx.logger.error(s"No query provided for task $taskName!") @@ -129,17 +144,14 @@ object PeriodicQueryTask { def withItems[F[_]: Sync]( ctx: Context[F, Args], store: Store[F], + search: OSearch[F], limit: Int, now: Timestamp )( cont: Vector[ListItem] => F[Unit] ): F[Unit] = - withQuery(ctx, store) { query => - val items = store - .transact(QItem.findItems(query, now.toUtcDate, 0, Batch.limit(limit))) - .compile - .to(Vector) - + withQuery(ctx, store, search) { qs => + val items = search.search(0, now.toUtcDate.some, Batch.limit(limit))(qs.q, qs.ftq) items.flatMap(cont) } diff --git a/modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala b/modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala index fa6a354c..9c552969 100644 --- a/modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala +++ b/modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala @@ -84,7 +84,7 @@ object BasicData { ) for { items <- QItem - .findItems(q, now.toUtcDate, 25, Batch.limit(itemIds.size)) + .queryItems(q, now.toUtcDate, 25, Batch.limit(itemIds.size), None) .compile .to(Vector) } yield items.map(apply(now)) diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 6501e499..a6e8375c 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -70,9 +70,6 @@ docspell.server { # In order to keep this low, a limit can be defined here. max-note-length = 180 - feature-search-2 = true - - # This defines whether the classification form in the collective # settings is displayed or not. If all joex instances have document # classification disabled, it makes sense to hide its settings from diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index ce587fbd..38bd05da 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -19,12 +19,16 @@ import docspell.backend.ops.OUpload.{UploadData, UploadMeta, UploadResult} import docspell.backend.ops._ import docspell.common._ import docspell.common.syntax.all._ -import docspell.ftsclient.FtsResult import docspell.restapi.model._ -import docspell.restserver.conv.Conversions._ import docspell.restserver.http4s.ContentDisposition import docspell.store.qb.Batch -import docspell.store.queries.{AttachmentLight => QAttachmentLight, IdRefCount} +import docspell.store.queries.{ + AttachmentLight => QAttachmentLight, + FieldStats => QFieldStats, + ItemFieldValue => QItemFieldValue, + TagCount => QTagCount, + _ +} import docspell.store.records._ import docspell.store.{AddResult, UpdateResult} @@ -34,7 +38,7 @@ import org.log4s.Logger trait Conversions { - def mkSearchStats(sum: OItemSearch.SearchSummary): SearchStats = + def mkSearchStats(sum: SearchSummary): SearchStats = SearchStats( sum.count, mkTagCloud(sum.tags), @@ -53,7 +57,7 @@ trait Conversions { def mkFolderStats(fs: docspell.store.queries.FolderCount): FolderStats = FolderStats(fs.id, fs.name, mkIdName(fs.owner), fs.count) - def mkFieldStats(fs: docspell.store.queries.FieldStats): FieldStats = + def mkFieldStats(fs: QFieldStats): FieldStats = FieldStats( fs.field.id, fs.field.name, @@ -76,7 +80,7 @@ trait Conversions { mkTagCloud(d.tags) ) - def mkTagCloud(tags: List[OCollective.TagCount]) = + def mkTagCloud(tags: List[QTagCount]) = TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count))) def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) = @@ -144,7 +148,7 @@ trait Conversions { data.relatedItems.map(mkItemLight).toList ) - def mkItemFieldValue(v: OItemSearch.ItemFieldValue): ItemFieldValue = + def mkItemFieldValue(v: QItemFieldValue): ItemFieldValue = ItemFieldValue(v.fieldId, v.fieldName, v.fieldLabel, v.fieldType, v.value) def mkAttachment( @@ -173,28 +177,13 @@ trait Conversions { OItemSearch.CustomValue(v.field, v.value) def mkItemList( - v: Vector[OItemSearch.ListItem], + v: Vector[ListItem], batch: Batch, capped: Boolean ): ItemLightList = { val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7)) - def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup = - ItemLightGroup(g._1, g._2.map(mkItemLight).toList) - - val gs = - groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs, batch.limit, batch.offset, capped) - } - - def mkItemListFts( - v: Vector[OFulltext.FtsItem], - batch: Batch, - capped: Boolean - ): ItemLightList = { - val groups = v.groupBy(item => item.item.date.toUtcDate.toString.substring(0, 7)) - - def mkGroup(g: (String, Vector[OFulltext.FtsItem])): ItemLightGroup = + def mkGroup(g: (String, Vector[ListItem])): ItemLightGroup = ItemLightGroup(g._1, g._2.map(mkItemLight).toList) val gs = @@ -203,13 +192,13 @@ trait Conversions { } def mkItemListWithTags( - v: Vector[OItemSearch.ListItemWithTags], + v: Vector[ListItemWithTags], batch: Batch, capped: Boolean ): ItemLightList = { val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7)) - def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup = + def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup = ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList) val gs = @@ -217,50 +206,7 @@ trait Conversions { ItemLightList(gs, batch.limit, batch.offset, capped) } - def mkItemListWithTagsFts( - v: Vector[OFulltext.FtsItemWithTags], - batch: Batch, - capped: Boolean - ): ItemLightList = { - val groups = v.groupBy(ti => ti.item.item.date.toUtcDate.toString.substring(0, 7)) - - def mkGroup(g: (String, Vector[OFulltext.FtsItemWithTags])): ItemLightGroup = - ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList) - - val gs = - groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs, batch.limit, batch.offset, capped) - } - - def mkItemListWithTagsFtsPlain( - v: Vector[OFulltext.FtsItemWithTags], - batch: Batch, - capped: Boolean - ): ItemLightList = - if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped) - else - ItemLightList( - List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)), - batch.limit, - batch.offset, - capped - ) - - def mkItemListFtsPlain( - v: Vector[OFulltext.FtsItem], - batch: Batch, - capped: Boolean - ): ItemLightList = - if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped) - else - ItemLightList( - List(ItemLightGroup("Results", v.map(mkItemLight).toList)), - batch.limit, - batch.offset, - capped - ) - - def mkItemLight(i: OItemSearch.ListItem): ItemLight = + def mkItemLight(i: ListItem): ItemLight = ItemLight( i.id, i.name, @@ -282,13 +228,7 @@ trait Conversions { Nil // highlight ) - def mkItemLight(i: OFulltext.FtsItem): ItemLight = { - val il = mkItemLight(i.item) - val highlight = mkHighlight(i.ftsData) - il.copy(highlighting = highlight) - } - - def mkItemLightWithTags(i: OItemSearch.ListItemWithTags): ItemLight = + def mkItemLightWithTags(i: ListItemWithTags): ItemLight = mkItemLight(i.item) .copy( tags = i.tags.map(mkTag), @@ -300,22 +240,6 @@ trait Conversions { def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight = AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount) - def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = { - val il = mkItemLightWithTags(i.item) - val highlight = mkHighlight(i.ftsData) - il.copy(highlighting = highlight) - } - - private def mkHighlight(ftsData: OFulltext.FtsData): List[HighlightEntry] = - ftsData.items.filter(_.context.nonEmpty).sortBy(-_.score).map { fdi => - fdi.matchData match { - case FtsResult.AttachmentData(_, aName) => - HighlightEntry(aName, fdi.context) - case FtsResult.ItemData => - HighlightEntry("Item", fdi.context) - } - } - // job def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = { @@ -571,7 +495,7 @@ trait Conversions { oid: Option[Ident], pid: Option[Ident] ): F[RContact] = - timeId.map { case (id, now) => + Conversions.timeId.map { case (id, now) => RContact(id, c.value.trim, c.kind, pid, oid, now) } @@ -590,7 +514,7 @@ trait Conversions { ) def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = - timeId.map { case (id, now) => + Conversions.timeId.map { case (id, now) => RUser( id, u.login, @@ -625,7 +549,7 @@ trait Conversions { Tag(rt.tagId, rt.name, rt.category, rt.created) def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = - timeId.map { case (id, now) => + Conversions.timeId.map { case (id, now) => RTag(id, cid, t.name.trim, t.category.map(_.trim), now) } @@ -653,7 +577,7 @@ trait Conversions { ) def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = - timeId.map { case (id, now) => + Conversions.timeId.map { case (id, now) => RSource( id, cid, @@ -691,7 +615,7 @@ trait Conversions { Equipment(re.eid, re.name, re.created, re.notes, re.use) def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = - timeId.map { case (id, now) => + Conversions.timeId.map { case (id, now) => REquipment(id, cid, e.name.trim, now, now, e.notes, e.use) } @@ -785,7 +709,7 @@ trait Conversions { header.mediaType.mainType, header.mediaType.subType, None - ).withCharsetName(header.mediaType.extensions.get("charset").getOrElse("unknown")) + ).withCharsetName(header.mediaType.extensions.getOrElse("charset", "unknown")) } object Conversions extends Conversions { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index bf182725..14dd23c9 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -13,12 +13,7 @@ import cats.implicits._ import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} -import docspell.backend.ops.OItemSearch.{Batch, Query} -import docspell.backend.ops.OSimpleSearch -import docspell.backend.ops.OSimpleSearch.StringSearchResult import docspell.common._ -import docspell.query.FulltextExtract.Result.TooMany -import docspell.query.FulltextExtract.Result.UnsupportedPosition import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -27,11 +22,11 @@ import docspell.restserver.http4s.ClientRequestInfo import docspell.restserver.http4s.Responses import docspell.restserver.http4s.{QueryParam => QP} +import org.http4s.HttpRoutes import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ import org.http4s.dsl.Http4sDsl import org.http4s.headers._ -import org.http4s.{HttpRoutes, Response} object ItemRoutes { def apply[F[_]: Async]( @@ -40,75 +35,12 @@ object ItemRoutes { user: AuthToken ): HttpRoutes[F] = { val logger = docspell.logging.getLogger[F] - val searchPart = ItemSearchPart[F](backend, cfg, user) + val searchPart = ItemSearchPart[F](backend.search, cfg, user) val dsl = new Http4sDsl[F] {} import dsl._ searchPart.routes <+> HttpRoutes.of { - case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( - offset - ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => - val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize)) - .restrictLimitTo(cfg.maxItemPageSize) - val limitCapped = limit.exists(_ > cfg.maxItemPageSize) - val itemQuery = ItemQueryString(q) - val settings = OSimpleSearch.Settings( - batch, - cfg.fullTextSearch.enabled, - detailFlag.getOrElse(false), - cfg.maxNoteLength, - searchMode.getOrElse(SearchMode.Normal) - ) - val fixQuery = Query.Fix(user.account, None, None) - searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped) - - case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => - val itemQuery = ItemQueryString(q) - val fixQuery = Query.Fix(user.account, None, None) - val settings = OSimpleSearch.StatsSettings( - useFTS = cfg.fullTextSearch.enabled, - searchMode = searchMode.getOrElse(SearchMode.Normal) - ) - searchItemStats(backend, dsl)(settings, fixQuery, itemQuery) - - case req @ POST -> Root / "search" => - for { - timed <- Duration.stopTime[F] - userQuery <- req.as[ItemQuery] - batch = Batch( - userQuery.offset.getOrElse(0), - userQuery.limit.getOrElse(cfg.maxItemPageSize) - ).restrictLimitTo( - cfg.maxItemPageSize - ) - limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize) - itemQuery = ItemQueryString(userQuery.query) - settings = OSimpleSearch.Settings( - batch, - cfg.fullTextSearch.enabled, - userQuery.withDetails.getOrElse(false), - cfg.maxNoteLength, - searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal) - ) - fixQuery = Query.Fix(user.account, None, None) - resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped) - dur <- timed - _ <- logger.debug(s"Search request: ${dur.formatExact}") - } yield resp - - case req @ POST -> Root / "searchStats" => - for { - userQuery <- req.as[ItemQuery] - itemQuery = ItemQueryString(userQuery.query) - fixQuery = Query.Fix(user.account, None, None) - settings = OSimpleSearch.StatsSettings( - useFTS = cfg.fullTextSearch.enabled, - searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal) - ) - resp <- searchItemStats(backend, dsl)(settings, fixQuery, itemQuery) - } yield resp - case GET -> Root / Ident(id) => for { item <- backend.itemSearch.findItem(id, user.account.collective) @@ -412,82 +344,6 @@ object ItemRoutes { } } - def searchItems[F[_]: Sync]( - backend: BackendApp[F], - dsl: Http4sDsl[F] - )( - settings: OSimpleSearch.Settings, - fixQuery: Query.Fix, - itemQuery: ItemQueryString, - limitCapped: Boolean - ): F[Response[F]] = { - import dsl._ - - def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList = - if (res.indexOnly) - Conversions.mkItemListFtsPlain(res.items, settings.batch, limitCapped) - else Conversions.mkItemListFts(res.items, settings.batch, limitCapped) - - def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList = - if (res.indexOnly) - Conversions.mkItemListWithTagsFtsPlain(res.items, settings.batch, limitCapped) - else Conversions.mkItemListWithTagsFts(res.items, settings.batch, limitCapped) - - backend.simpleSearch - .searchByString(settings)(fixQuery, itemQuery) - .flatMap { - case StringSearchResult.Success(items) => - Ok( - items.fold( - convertFts, - convertFtsFull, - els => Conversions.mkItemList(els, settings.batch, limitCapped), - els => Conversions.mkItemListWithTags(els, settings.batch, limitCapped) - ) - ) - case StringSearchResult.FulltextMismatch(TooMany) => - BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) - case StringSearchResult.FulltextMismatch(UnsupportedPosition) => - BadRequest( - BasicResult( - false, - "Fulltext search must be in root position or inside the first AND." - ) - ) - case StringSearchResult.ParseFailed(pf) => - BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) - } - } - - def searchItemStats[F[_]: Sync]( - backend: BackendApp[F], - dsl: Http4sDsl[F] - )( - settings: OSimpleSearch.StatsSettings, - fixQuery: Query.Fix, - itemQuery: ItemQueryString - ): F[Response[F]] = { - import dsl._ - - backend.simpleSearch - .searchSummaryByString(settings)(fixQuery, itemQuery) - .flatMap { - case StringSearchResult.Success(summary) => - Ok(Conversions.mkSearchStats(summary)) - case StringSearchResult.FulltextMismatch(TooMany) => - BadRequest(BasicResult(false, "Only one fulltext search term is allowed.")) - case StringSearchResult.FulltextMismatch(UnsupportedPosition) => - BadRequest( - BasicResult( - false, - "Fulltext search must be in root position or inside the first AND." - ) - ) - case StringSearchResult.ParseFailed(pf) => - BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) - } - } - implicit final class OptionString(opt: Option[String]) { def notEmpty: Option[String] = opt.map(_.trim).filter(_.nonEmpty) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala index bd386b90..00775d94 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala @@ -11,26 +11,30 @@ import java.time.LocalDate import cats.effect._ import cats.syntax.all._ -import docspell.backend.BackendApp import docspell.backend.auth.AuthToken -import docspell.backend.ops.search.QueryParseResult -import docspell.common.{Duration, SearchMode, Timestamp} +import docspell.backend.ops.OShare +import docspell.backend.ops.OShare.ShareQuery +import docspell.backend.ops.search.{OSearch, QueryParseResult} +import docspell.common._ import docspell.query.FulltextExtract.Result import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions import docspell.restserver.http4s.{QueryParam => QP} import docspell.store.qb.Batch -import docspell.store.queries.ListItemWithTags +import docspell.store.queries.{ListItemWithTags, SearchSummary} import org.http4s.circe.CirceEntityCodec._ import org.http4s.dsl.Http4sDsl import org.http4s.{HttpRoutes, Response} final class ItemSearchPart[F[_]: Async]( - backend: BackendApp[F], + searchOps: OSearch[F], cfg: Config, - authToken: AuthToken + parseQuery: (SearchMode, String) => QueryParseResult, + changeSummary: SearchSummary => SearchSummary = identity, + searchPath: String = "search", + searchStatsPath: String = "searchStats" ) extends Http4sDsl[F] { private[this] val logger = docspell.logging.getLogger[F] @@ -39,9 +43,9 @@ final class ItemSearchPart[F[_]: Async]( if (!cfg.featureSearch2) HttpRoutes.empty else HttpRoutes.of { - case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( - offset - ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => + case GET -> Root / `searchPath` :? QP.Query(q) :? QP.Limit(limit) :? + QP.Offset(offset) :? QP.WithDetails(detailFlag) :? + QP.SearchKind(searchMode) => val userQuery = ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse("")) for { @@ -49,7 +53,7 @@ final class ItemSearchPart[F[_]: Async]( resp <- search(userQuery, today) } yield resp - case req @ POST -> Root / "search" => + case req @ POST -> Root / `searchPath` => for { timed <- Duration.stopTime[F] userQuery <- req.as[ItemQuery] @@ -59,14 +63,15 @@ final class ItemSearchPart[F[_]: Async]( _ <- logger.debug(s"Search request: ${dur.formatExact}") } yield resp - case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => + case GET -> Root / `searchStatsPath` :? QP.Query(q) :? + QP.SearchKind(searchMode) => val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse("")) for { today <- Timestamp.current[F].map(_.toUtcDate) resp <- searchStats(userQuery, today) } yield resp - case req @ POST -> Root / "searchStats" => + case req @ POST -> Root / `searchStatsPath` => for { timed <- Duration.stopTime[F] userQuery <- req.as[ItemQuery] @@ -84,8 +89,8 @@ final class ItemSearchPart[F[_]: Async]( identity, res => for { - summary <- backend.search.searchSummary(today)(res.q, res.ftq) - resp <- Ok(Conversions.mkSearchStats(summary)) + summary <- searchOps.searchSummary(today.some)(res.q, res.ftq) + resp <- Ok(Conversions.mkSearchStats(changeSummary(summary))) } yield resp ) } @@ -103,8 +108,9 @@ final class ItemSearchPart[F[_]: Async]( identity, res => for { - items <- backend.search - .searchSelect(details, cfg.maxNoteLength, today, batch)( + _ <- logger.warn(s"Searching with query: $res") + items <- searchOps + .searchSelect(details, cfg.maxNoteLength, today.some, batch)( res.q, res.ftq ) @@ -122,29 +128,10 @@ final class ItemSearchPart[F[_]: Async]( userQuery: ItemQuery, mode: SearchMode ): Either[F[Response[F]], QueryParseResult.Success] = - backend.search.parseQueryString(authToken.account, mode, userQuery.query) match { - case s: QueryParseResult.Success => - Right(s.withFtsEnabled(cfg.fullTextSearch.enabled)) - - case QueryParseResult.ParseFailed(err) => - Left(BadRequest(BasicResult(false, s"Invalid query: $err"))) - - case QueryParseResult.FulltextMismatch(Result.TooMany) => - Left( - BadRequest( - BasicResult(false, "Only one fulltext search expression is allowed.") - ) - ) - case QueryParseResult.FulltextMismatch(Result.UnsupportedPosition) => - Left( - BadRequest( - BasicResult( - false, - "A fulltext search may only appear in the root and expression." - ) - ) - ) - } + convertParseResult( + parseQuery(mode, userQuery.query) + .map(_.withFtsEnabled(cfg.fullTextSearch.enabled)) + ) def convert( items: Vector[ListItemWithTags], @@ -202,13 +189,56 @@ final class ItemSearchPart[F[_]: Async]( Nil } ) + + def convertParseResult( + r: QueryParseResult + ): Either[F[Response[F]], QueryParseResult.Success] = + r match { + case s: QueryParseResult.Success => + Right(s) + + case QueryParseResult.ParseFailed(err) => + BadRequest(BasicResult(false, s"Invalid query: $err")).asLeft + + case QueryParseResult.FulltextMismatch(Result.TooMany) => + BadRequest( + BasicResult(false, "Only one fulltext search expression is allowed.") + ).asLeft + + case QueryParseResult.FulltextMismatch(Result.UnsupportedPosition) => + BadRequest( + BasicResult( + false, + "A fulltext search may only appear in the root and expression." + ) + ).asLeft + } } object ItemSearchPart { def apply[F[_]: Async]( - backend: BackendApp[F], + search: OSearch[F], cfg: Config, token: AuthToken ): ItemSearchPart[F] = - new ItemSearchPart[F](backend, cfg, token) + new ItemSearchPart[F]( + search, + cfg, + (m, s) => search.parseQueryString(token.account, m, s) + ) + + def apply[F[_]: Async]( + search: OSearch[F], + share: OShare[F], + cfg: Config, + shareQuery: ShareQuery + ): ItemSearchPart[F] = + new ItemSearchPart[F]( + search, + cfg, + (_, s) => share.parseQuery(shareQuery, s), + changeSummary = _.onlyExisting, + searchPath = "query", + searchStatsPath = "stats" + ) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala index 92a66ff1..5a13df87 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala @@ -6,25 +6,14 @@ package docspell.restserver.routes +import cats.data.Kleisli import cats.effect._ -import cats.implicits._ import docspell.backend.BackendApp import docspell.backend.auth.ShareToken -import docspell.backend.ops.OSimpleSearch -import docspell.backend.ops.OSimpleSearch.StringSearchResult -import docspell.common._ -import docspell.query.FulltextExtract.Result.{TooMany, UnsupportedPosition} -import docspell.restapi.model._ import docspell.restserver.Config -import docspell.restserver.conv.Conversions -import docspell.store.qb.Batch -import docspell.store.queries.{Query, SearchSummary} -import org.http4s.circe.CirceEntityDecoder._ -import org.http4s.circe.CirceEntityEncoder._ -import org.http4s.dsl.Http4sDsl -import org.http4s.{HttpRoutes, Response} +import org.http4s.HttpRoutes object ShareSearchRoutes { @@ -32,80 +21,13 @@ object ShareSearchRoutes { backend: BackendApp[F], cfg: Config, token: ShareToken - ): HttpRoutes[F] = { - val logger = docspell.logging.getLogger[F] - - val dsl = new Http4sDsl[F] {} - import dsl._ - - HttpRoutes.of { - case req @ POST -> Root / "query" => - backend.share - .findShareQuery(token.id) - .semiflatMap { share => - for { - userQuery <- req.as[ItemQuery] - batch = Batch( - userQuery.offset.getOrElse(0), - userQuery.limit.getOrElse(cfg.maxItemPageSize) - ).restrictLimitTo( - cfg.maxItemPageSize - ) - limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize) - itemQuery = ItemQueryString(userQuery.query) - settings = OSimpleSearch.Settings( - batch, - cfg.fullTextSearch.enabled, - userQuery.withDetails.getOrElse(false), - cfg.maxNoteLength, - searchMode = SearchMode.Normal - ) - account = share.account - fixQuery = Query.Fix(account, Some(share.query.expr), None) - _ <- logger.debug(s"Searching in share ${share.id.id}: ${userQuery.query}") - resp <- ItemRoutes.searchItems(backend, dsl)( - settings, - fixQuery, - itemQuery, - limitCapped - ) - } yield resp - } - .getOrElseF(NotFound()) - - case req @ POST -> Root / "stats" => - for { - userQuery <- req.as[ItemQuery] - itemQuery = ItemQueryString(userQuery.query) - settings = OSimpleSearch.StatsSettings( - useFTS = cfg.fullTextSearch.enabled, - searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal) - ) - stats <- backend.share.searchSummary(settings)(token.id, itemQuery).value - resp <- stats.map(mkSummaryResponse(dsl)).getOrElse(NotFound()) - } yield resp + ): HttpRoutes[F] = + Kleisli { req => + for { + shareQuery <- backend.share.findShareQuery(token.id) + searchPart = ItemSearchPart(backend.search, backend.share, cfg, shareQuery) + routes = searchPart.routes + resp <- routes(req) + } yield resp } - } - - def mkSummaryResponse[F[_]: Sync]( - dsl: Http4sDsl[F] - )(r: StringSearchResult[SearchSummary]): F[Response[F]] = { - import dsl._ - r match { - case StringSearchResult.Success(summary) => - Ok(Conversions.mkSearchStats(summary)) - case StringSearchResult.FulltextMismatch(TooMany) => - BadRequest(BasicResult(false, "Fulltext search is not possible in this share.")) - case StringSearchResult.FulltextMismatch(UnsupportedPosition) => - BadRequest( - BasicResult( - false, - "Fulltext search must be in root position or inside the first AND." - ) - ) - case StringSearchResult.ParseFailed(pf) => - BadRequest(BasicResult(false, s"Error reading query: ${pf.render}")) - } - } - } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 0568ac2a..44054354 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -73,14 +73,6 @@ object QItem extends FtsSupport { sql.query[ListItem].stream } - // ---- - - def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] = - Select(count(a.id).s, from(a), a.itemId.in(items)).build - .query[Int] - .unique - .map(_ + items.size) - def findItem(id: Ident, collective: Ident): ConnectionIO[Option[ItemData]] = { val ref = RItem.as("ref") val cq = @@ -314,14 +306,6 @@ object QItem extends FtsSupport { Condition.unit } - def findItems( - q: Query, - today: LocalDate, - maxNoteLen: Int, - batch: Batch - ): Stream[ConnectionIO, ListItem] = - queryItems(q, today, maxNoteLen, batch, None) - def searchStats(today: LocalDate, ftsTable: Option[RFtsResult.Table])( q: Query ): ConnectionIO[SearchSummary] = @@ -524,47 +508,6 @@ object QItem extends FtsSupport { } } - def findSelectedItems( - q: Query, - today: LocalDate, - maxNoteLen: Int, - items: Set[SelectedItem] - ): Stream[ConnectionIO, ListItem] = - if (items.isEmpty) Stream.empty - else { - val i = RItem.as("i") - - object Tids extends TableDef { - val tableName = "tids" - val alias = Some("tw") - val itemId = Column[Ident]("item_id", this) - val weight = Column[Double]("weight", this) - val all = Vector[Column[_]](itemId, weight) - } - - val cte = - CteBind( - Tids, - Tids.all, - Select.RawSelect( - fr"VALUES" ++ - items - .map(it => fr"(${it.itemId}, ${it.weight})") - .reduce((r, e) => r ++ fr"," ++ e) - ) - ) - - val from = findItemsBase(q.fix, today, maxNoteLen, None) - .appendCte(cte) - .appendSelect(Tids.weight.s) - .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id)) - .orderBy(Tids.weight.desc) - .build - - logger.stream.trace(s"fts query: $from").drain ++ - from.query[ListItem].stream - } - /** Same as `findItems` but resolves the tags for each item. Note that this is * implemented by running an additional query per item. */ diff --git a/modules/store/src/main/scala/docspell/store/queries/Query.scala b/modules/store/src/main/scala/docspell/store/queries/Query.scala index f8854b05..98c86d23 100644 --- a/modules/store/src/main/scala/docspell/store/queries/Query.scala +++ b/modules/store/src/main/scala/docspell/store/queries/Query.scala @@ -17,6 +17,15 @@ case class Query(fix: Query.Fix, cond: Query.QueryCond) { def withCond(f: Query.QueryCond => Query.QueryCond): Query = copy(cond = f(cond)) + def andCond(c: ItemQuery.Expr): Query = + withCond { + case Query.QueryExpr(Some(q)) => + Query.QueryExpr(ItemQuery.Expr.and(q, c)) + + case Query.QueryExpr(None) => + Query.QueryExpr(c) + } + def withOrder(orderAsc: RItem.Table => Column[_]): Query = withFix(_.copy(order = Some(_.byItemColumnAsc(orderAsc))))