TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused code (search update)

Browse Source
This commit is contained in:
eikek
2022-06-05 22:06:22 +02:00
parent c6a9a17f89
commit 7ce6bc2f9d
19 changed files with 228 additions and 1157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
modules/backend/src/main/scala/docspell/backend/BackendApp.scala
View File

@ -47,7 +47,6 @@ trait BackendApp[F[_]] {
def userTask: OUserTask[F]
def folder: OFolder[F]
def customFields: OCustomFields[F]
def simpleSearch: OSimpleSearch[F]
def clientSettings: OClientSettings[F]
def totp: OTotp[F]
def share: OShare[F]
@ -99,8 +98,6 @@ object BackendApp {
itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs)
itemSearchImpl <- OItemSearch(store)
fulltextImpl <- OFulltext(
itemSearchImpl,
ftsClient,
store,
schedulerModule.jobs
)
@ -112,15 +109,15 @@ object BackendApp {
)
folderImpl <- OFolder(store)
customFieldsImpl <- OCustomFields(store)
simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl)
clientSettingsImpl <- OClientSettings(store)
searchImpl <- Resource.pure(OSearch(store, ftsClient))
shareImpl <- Resource.pure(
OShare(store, itemSearchImpl, simpleSearchImpl, javaEmil)
OShare(store, itemSearchImpl, searchImpl, javaEmil)
)
notifyImpl <- ONotification(store, notificationMod)
bookmarksImpl <- OQueryBookmarks(store)
fileRepoImpl <- OFileRepository(store, schedulerModule.jobs)
itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl))
itemLinkImpl <- Resource.pure(OItemLink(store, searchImpl))
downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs))
attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs))
addonsImpl <- Resource.pure(
@ -132,7 +129,6 @@ object BackendApp {
joexImpl
)
)
searchImpl <- Resource.pure(OSearch(store, ftsClient))
} yield new BackendApp[F] {
val pubSub = pubSubT
val login = loginImpl
@ -153,7 +149,6 @@ object BackendApp {
val userTask = userTaskImpl
val folder = folderImpl
val customFields = customFieldsImpl
val simpleSearch = simpleSearchImpl
val clientSettings = clientSettingsImpl
val totp = totpImpl
val share = shareImpl

281
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@ -6,46 +6,17 @@
package docspell.backend.ops
import cats.data.NonEmptyList
import cats.effect._
import cats.implicits._
import fs2.Stream
import docspell.backend.JobFactory
import docspell.backend.ops.OItemSearch._
import docspell.common._
import docspell.ftsclient._
import docspell.query.ItemQuery._
import docspell.query.ItemQueryDsl._
import docspell.scheduler.JobStore
import docspell.store.queries.{QFolder, QItem, SelectedItem}
import docspell.store.Store
import docspell.store.records.RJob
import docspell.store.{Store, qb}
trait OFulltext[F[_]] {
def findItems(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: qb.Batch
): F[Vector[OFulltext.FtsItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnly(maxNoteLen: Int)(
fts: OFulltext.FtsInput,
account: AccountId,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
/** Clears the full-text index completely and launches a task that indexes all data. */
def reindexAll: F[Unit]
@ -56,30 +27,7 @@ trait OFulltext[F[_]] {
}
object OFulltext {
case class FtsInput(
query: String,
highlightPre: String = "***",
highlightPost: String = "***"
)
case class FtsDataItem(
score: Double,
matchData: FtsResult.MatchData,
context: List[String]
)
case class FtsData(
maxScore: Double,
count: Int,
qtime: Duration,
items: List[FtsDataItem]
)
case class FtsItem(item: ListItem, ftsData: FtsData)
case class FtsItemWithTags(item: ListItemWithTags, ftsData: FtsData)
def apply[F[_]: Async](
itemSearch: OItemSearch[F],
fts: FtsClient[F],
store: Store[F],
jobStore: JobStore[F]
): Resource[F, OFulltext[F]] =
@ -103,232 +51,5 @@ object OFulltext {
if (exist.isDefined) ().pure[F]
else jobStore.insertIfNew(job.encode)
} yield ()
def findIndexOnly(maxNoteLen: Int)(
ftsQ: OFulltext.FtsInput,
account: AccountId,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]] = {
val fq = FtsQuery(
ftsQ.query,
account.collective,
Set.empty,
Set.empty,
batch.limit,
batch.offset,
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
)
for {
_ <- logger.trace(s"Find index only: ${ftsQ.query}/$batch")
folders <- store.transact(QFolder.getMemberFolders(account))
ftsR <- fts.search(fq.withFolders(folders))
ftsItems = ftsR.results.groupBy(_.itemId)
select =
ftsItems.values
.map(_.minBy(-_.score))
.map(r => SelectedItem(r.itemId, r.score))
.toSet
now <- Timestamp.current[F]
itemsWithTags <-
store
.transact(
QItem.findItemsWithTags(
account.collective,
QItem.findSelectedItems(
Query.all(account),
now.toUtcDate,
maxNoteLen,
select
)
)
)
.take(batch.limit.toLong)
.compile
.toVector
res =
itemsWithTags
.collect(convertFtsData(ftsR, ftsItems))
.map { case (li, fd) => FtsItemWithTags(li, fd) }
} yield res
}
def findIndexOnlySummary(
account: AccountId,
ftsQ: OFulltext.FtsInput
): F[SearchSummary] = {
val fq = FtsQuery(
ftsQ.query,
account.collective,
Set.empty,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
for {
folder <- store.transact(QFolder.getMemberFolders(account))
now <- Timestamp.current[F]
itemIds <- fts
.searchAll(fq.withFolders(folder))
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
itemIdsQuery = NonEmptyList
.fromList(itemIds.toList)
.map(ids => Attr.ItemId.in(ids.map(_.id)))
.getOrElse(Attr.ItemId.notExists)
q = Query
.all(account)
.withFix(_.copy(query = itemIdsQuery.some))
res <- store.transact(QItem.searchStats(now.toUtcDate, None)(q))
} yield res
}
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItems(maxNoteLen),
convertFtsData[ListItem]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map { case (li, fd) => FtsItem(li, fd) }
.compile
.toVector
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
ftsQ: FtsInput,
batch: qb.Batch
): F[Vector[FtsItemWithTags]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItemsWithTags(maxNoteLen),
convertFtsData[ListItemWithTags]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map { case (li, fd) => FtsItemWithTags(li, fd) }
.compile
.toVector
def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] =
for {
search <- itemSearch.findItems(0)(q, Batch.all)
fq = FtsQuery(
ftsQ.query,
q.fix.account.collective,
search.map(_.id).toSet,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
items <- fts
.searchAll(fq)
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
itemIdsQuery = NonEmptyList
.fromList(items.toList)
.map(ids => Attr.ItemId.in(ids.map(_.id)))
.getOrElse(Attr.ItemId.notExists)
qnext = q.withFix(_.copy(query = itemIdsQuery.some))
now <- Timestamp.current[F]
res <- store.transact(QItem.searchStats(now.toUtcDate, None)(qnext))
} yield res
// Helper
private def findItemsFts[A: ItemId, B](
q: Query,
ftsQ: FtsInput,
batch: qb.Batch,
search: (Query, qb.Batch) => F[Vector[A]],
convert: (
FtsResult,
Map[Ident, List[FtsResult.ItemMatch]]
) => PartialFunction[A, (A, FtsData)]
): Stream[F, (A, FtsData)] =
findItemsFts0(q, ftsQ, batch, search, convert)
.takeThrough(_._1 >= batch.limit)
.flatMap(x => Stream.emits(x._2))
private def findItemsFts0[A: ItemId, B](
q: Query,
ftsQ: FtsInput,
batch: qb.Batch,
search: (Query, qb.Batch) => F[Vector[A]],
convert: (
FtsResult,
Map[Ident, List[FtsResult.ItemMatch]]
) => PartialFunction[A, (A, FtsData)]
): Stream[F, (Int, Vector[(A, FtsData)])] = {
val sqlResult = search(q, batch)
val fq = FtsQuery(
ftsQ.query,
q.fix.account.collective,
Set.empty,
Set.empty,
0,
0,
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
)
val qres =
for {
items <- sqlResult
ids = items.map(a => ItemId[A].itemId(a))
idsNel = NonEmptyList.fromFoldable(ids)
// must find all index results involving the items.
// Currently there is one result per item + one result per
// attachment
limit <- idsNel
.map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds)))
.getOrElse(0.pure[F])
ftsQ = fq.copy(items = ids.toSet, limit = limit)
ftsR <- fts.search(ftsQ)
ftsItems = ftsR.results.groupBy(_.itemId)
res = items.collect(convert(ftsR, ftsItems))
} yield (items.size, res)
Stream.eval(qres) ++ findItemsFts0(q, ftsQ, batch.next, search, convert)
}
private def convertFtsData[A: ItemId](
ftr: FtsResult,
ftrItems: Map[Ident, List[FtsResult.ItemMatch]]
): PartialFunction[A, (A, FtsData)] = {
case a if ftrItems.contains(ItemId[A].itemId(a)) =>
val ftsDataItems = ftrItems
.getOrElse(ItemId[A].itemId(a), Nil)
.map(im =>
FtsDataItem(im.score, im.data, ftr.highlight.getOrElse(im.id, Nil))
)
(a, FtsData(ftr.maxScore, ftr.count, ftr.qtime, ftsDataItems))
}
})
trait ItemId[A] {
def itemId(a: A): Ident
}
object ItemId {
def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev
def from[A](f: A => Ident): ItemId[A] =
(a: A) => f(a)
implicit val listItemId: ItemId[ListItem] =
ItemId.from(_.id)
implicit val listItemWithTagsId: ItemId[ListItemWithTags] =
ItemId.from(_.item.id)
}
}

13
modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala
View File

@ -11,11 +11,12 @@ import cats.effect._
import cats.implicits._
import docspell.backend.ops.OItemLink.LinkResult
import docspell.backend.ops.search.OSearch
import docspell.common.{AccountId, Ident}
import docspell.query.ItemQuery
import docspell.query.ItemQueryDsl._
import docspell.store.qb.Batch
import docspell.store.queries.Query
import docspell.store.queries.{ListItemWithTags, Query}
import docspell.store.records.RItemLink
import docspell.store.{AddResult, Store}
@ -29,7 +30,7 @@ trait OItemLink[F[_]] {
account: AccountId,
item: Ident,
batch: Batch
): F[Vector[OItemSearch.ListItemWithTags]]
): F[Vector[ListItemWithTags]]
}
object OItemLink {
@ -44,13 +45,13 @@ object OItemLink {
def linkTargetItemError: LinkResult = LinkTargetItemError
}
def apply[F[_]: Sync](store: Store[F], search: OItemSearch[F]): OItemLink[F] =
def apply[F[_]: Sync](store: Store[F], search: OSearch[F]): OItemLink[F] =
new OItemLink[F] {
def getRelated(
accountId: AccountId,
item: Ident,
batch: Batch
): F[Vector[OItemSearch.ListItemWithTags]] =
): F[Vector[ListItemWithTags]] =
store
.transact(RItemLink.findLinked(accountId.collective, item))
.map(ids => NonEmptyList.fromList(ids.toList))
@ -62,10 +63,10 @@ object OItemLink {
.Fix(accountId, Some(ItemQuery.Expr.ValidItemStates), None),
Query.QueryExpr(expr)
)
search.findItemsWithTags(0)(query, batch)
search.searchWithDetails(0, None, batch)(query, None)
case None =>
Vector.empty[OItemSearch.ListItemWithTags].pure[F]
Vector.empty[ListItemWithTags].pure[F]
}
def addAll(cid: Ident, target: Ident, related: NonEmptyList[Ident]): F[LinkResult] =

54
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@ -25,15 +25,6 @@ trait OItemSearch[F[_]] {
def findDeleted(collective: Ident, maxUpdate: Timestamp, limit: Int): F[Vector[RItem]]
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findItemsSummary(q: Query): F[SearchSummary]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
def findAttachmentSource(
@ -63,9 +54,6 @@ trait OItemSearch[F[_]] {
object OItemSearch {
type SearchSummary = queries.SearchSummary
val SearchSummary = queries.SearchSummary
type CustomValue = queries.CustomValue
val CustomValue = queries.CustomValue
@ -75,12 +63,6 @@ object OItemSearch {
type Batch = qb.Batch
val Batch = docspell.store.qb.Batch
type ListItem = queries.ListItem
val ListItem = queries.ListItem
type ListItemWithTags = queries.ListItemWithTags
val ListItemWithTags = queries.ListItemWithTags
type ItemFieldValue = queries.ItemFieldValue
val ItemFieldValue = queries.ItemFieldValue
@ -136,19 +118,6 @@ object OItemSearch {
store
.transact(QItem.findItem(id, collective))
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap { today =>
store
.transact(
QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong)
)
.compile
.toVector
}
def findDeleted(
collective: Ident,
maxUpdate: Timestamp,
@ -160,28 +129,6 @@ object OItemSearch {
.compile
.toVector
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
for {
now <- Timestamp.current[F]
search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch)
res <- store
.transact(
QItem
.findItemsWithTags(q.fix.account.collective, search)
.take(batch.limit.toLong)
)
.compile
.toVector
} yield res
def findItemsSummary(q: Query): F[SearchSummary] =
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap(today => store.transact(QItem.searchStats(today, None)(q)))
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))
@ -298,6 +245,5 @@ object OItemSearch {
coll <- OptionT(RSource.findCollective(sourceId))
items <- OptionT.liftF(QItem.findByChecksum(checksum, coll, Set.empty))
} yield items).value)
})
}

37
modules/backend/src/main/scala/docspell/backend/ops/OShare.scala
View File

@ -14,13 +14,12 @@ import docspell.backend.PasswordCrypt
import docspell.backend.auth.ShareToken
import docspell.backend.ops.OItemSearch._
import docspell.backend.ops.OShare._
import docspell.backend.ops.OSimpleSearch.StringSearchResult
import docspell.backend.ops.search.{OSearch, QueryParseResult}
import docspell.common._
import docspell.query.ItemQuery.Expr
import docspell.query.ItemQuery.Expr.AttachId
import docspell.query.{FulltextExtract, ItemQuery}
import docspell.store.Store
import docspell.store.queries.SearchSummary
import docspell.store.records._
import emil._
@ -67,9 +66,10 @@ trait OShare[F[_]] {
def findItem(itemId: Ident, shareId: Ident): OptionT[F, ItemData]
def searchSummary(
settings: OSimpleSearch.StatsSettings
)(shareId: Ident, q: ItemQueryString): OptionT[F, StringSearchResult[SearchSummary]]
/** Parses a query and amends the result with the stored query of the share. The result
* can be used with [[OSearch]] to search for items.
*/
def parseQuery(share: ShareQuery, qs: String): QueryParseResult
def sendMail(account: AccountId, connection: Ident, mail: ShareMail): F[SendResult]
}
@ -148,7 +148,7 @@ object OShare {
def apply[F[_]: Async](
store: Store[F],
itemSearch: OItemSearch[F],
simpleSearch: OSimpleSearch[F],
search: OSearch[F],
emil: Emil[F]
): OShare[F] =
new OShare[F] {
@ -325,8 +325,8 @@ object OShare {
Query.QueryExpr(idExpr)
)
OptionT(
itemSearch
.findItems(0)(checkQuery, Batch.limit(1))
search
.search(0, None, Batch.limit(1))(checkQuery, None)
.map(_.headOption.map(_ => ()))
).flatTapNone(
logger.info(
@ -335,22 +335,11 @@ object OShare {
)
}
def searchSummary(
settings: OSimpleSearch.StatsSettings
)(
shareId: Ident,
q: ItemQueryString
): OptionT[F, StringSearchResult[SearchSummary]] =
findShareQuery(shareId)
.semiflatMap { share =>
val fix = Query.Fix(share.account, Some(share.query.expr), None)
simpleSearch
.searchSummaryByString(settings)(fix, q)
.map {
case StringSearchResult.Success(summary) =>
StringSearchResult.Success(summary.onlyExisting)
case other => other
}
def parseQuery(share: ShareQuery, qs: String): QueryParseResult =
search
.parseQueryString(share.account, SearchMode.Normal, qs)
.map { case QueryParseResult.Success(q, ftq) =>
QueryParseResult.Success(q.withFix(_.andQuery(share.query.expr)), ftq)
}
def sendMail(

300
modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala
View File

@ -1,300 +0,0 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.backend.ops
import cats.Applicative
import cats.effect.Sync
import cats.implicits._
import docspell.backend.ops.OSimpleSearch._
import docspell.common._
import docspell.query._
import docspell.store.qb.Batch
import docspell.store.queries.Query
import docspell.store.queries.SearchSummary
import org.log4s.getLogger
/** A "porcelain" api on top of OFulltext and OItemSearch. This takes care of restricting
* the items to a subset, e.g. only items that have a "valid" state.
*/
trait OSimpleSearch[F[_]] {
/** Search for items using the given query and optional fulltext search.
*
* When using fulltext search only (the query is empty), only the index is searched. It
* is assumed that the index doesn't contain "invalid" items. When using a query, then
* a condition to select only valid items is added to it.
*/
def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
/** Using the same arguments as in `search`, this returns a summary and not the results.
*/
def searchSummary(
settings: StatsSettings
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
/** Calls `search` by parsing the given query string into a query that is then amended
* wtih the given `fix` query.
*/
final def searchByString(
settings: Settings
)(fix: Query.Fix, q: ItemQueryString)(implicit
F: Applicative[F]
): F[StringSearchResult[Items]] =
OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts))
/** Same as `searchByString` but returning a summary instead of the results. */
final def searchSummaryByString(
settings: StatsSettings
)(fix: Query.Fix, q: ItemQueryString)(implicit
F: Applicative[F]
): F[StringSearchResult[SearchSummary]] =
OSimpleSearch.applySearch[F, SearchSummary](fix, q)((iq, fts) =>
searchSummary(settings)(iq, fts)
)
}
object OSimpleSearch {
private[this] val logger = getLogger
sealed trait StringSearchResult[+A]
object StringSearchResult {
case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing]
def parseFailed[A](error: ParseFailure): StringSearchResult[A] =
ParseFailed(error)
case class FulltextMismatch(error: FulltextExtract.FailureResult)
extends StringSearchResult[Nothing]
def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] =
FulltextMismatch(error)
case class Success[A](value: A) extends StringSearchResult[A]
}
final case class Settings(
batch: Batch,
useFTS: Boolean,
resolveDetails: Boolean,
maxNoteLen: Int,
searchMode: SearchMode
)
final case class StatsSettings(
useFTS: Boolean,
searchMode: SearchMode
)
sealed trait Items {
def fold[A](
f1: Items.FtsItems => A,
f2: Items.FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A
}
object Items {
def ftsItems(indexOnly: Boolean)(items: Vector[OFulltext.FtsItem]): Items =
FtsItems(items, indexOnly)
case class FtsItems(items: Vector[OFulltext.FtsItem], indexOnly: Boolean)
extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f1(this)
}
def ftsItemsFull(indexOnly: Boolean)(
items: Vector[OFulltext.FtsItemWithTags]
): Items =
FtsItemsFull(items, indexOnly)
case class FtsItemsFull(items: Vector[OFulltext.FtsItemWithTags], indexOnly: Boolean)
extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f2(this)
}
def itemsPlain(items: Vector[OItemSearch.ListItem]): Items =
ItemsPlain(items)
case class ItemsPlain(items: Vector[OItemSearch.ListItem]) extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f3(items)
}
def itemsFull(items: Vector[OItemSearch.ListItemWithTags]): Items =
ItemsFull(items)
case class ItemsFull(items: Vector[OItemSearch.ListItemWithTags]) extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f4(items)
}
}
def apply[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]): OSimpleSearch[F] =
new Impl(fts, is)
/** Parses the query and calls `run` with the result, which searches items. */
private def applySearch[F[_]: Applicative, A](fix: Query.Fix, q: ItemQueryString)(
run: (Query, Option[String]) => F[A]
): F[StringSearchResult[A]] = {
val parsed: Either[StringSearchResult[A], Option[ItemQuery]] =
if (q.isEmpty) Right(None)
else
ItemQueryParser
.parse(q.query)
.leftMap(StringSearchResult.parseFailed)
.map(_.some)
def makeQuery(itemQuery: Option[ItemQuery]): F[StringSearchResult[A]] =
runQuery[F, A](itemQuery) {
case Some(s) =>
run(Query(fix, Query.QueryExpr(s.getExprPart)), s.getFulltextPart)
case None =>
run(Query(fix), None)
}
parsed match {
case Right(iq) =>
makeQuery(iq)
case Left(err) =>
err.pure[F]
}
}
/** Calls `run` with one of the success results when extracting the fulltext search node
* from the query.
*/
private def runQuery[F[_]: Applicative, A](
itemQuery: Option[ItemQuery]
)(run: Option[FulltextExtract.SuccessResult] => F[A]): F[StringSearchResult[A]] =
itemQuery match {
case Some(iq) =>
iq.findFulltext match {
case s: FulltextExtract.SuccessResult =>
run(Some(s)).map(StringSearchResult.Success.apply)
case other: FulltextExtract.FailureResult =>
StringSearchResult.fulltextMismatch[A](other).pure[F]
}
case None =>
run(None).map(StringSearchResult.Success.apply)
}
final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F])
extends OSimpleSearch[F] {
/** Implements searching like this: it exploits the fact that teh fulltext index only
* contains valid items. When searching via sql the query expression selecting only
* valid items is added here.
*/
def search(
settings: Settings
)(q: Query, fulltextQuery: Option[String]): F[Items] = {
// 1. fulltext only if fulltextQuery.isDefined && q.isEmpty && useFTS
// 2. sql+fulltext if fulltextQuery.isDefined && q.nonEmpty && useFTS
// 3. sql-only else (if fulltextQuery.isEmpty || !useFTS)
val validItemQuery =
settings.searchMode match {
case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed))
case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates))
case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed))
}
fulltextQuery match {
case Some(ftq) if settings.useFTS =>
if (q.isEmpty) {
logger.debug(s"Using index only search: $fulltextQuery")
if (settings.searchMode == SearchMode.Trashed)
Items.ftsItemsFull(true)(Vector.empty).pure[F]
else
fts
.findIndexOnly(settings.maxNoteLen)(
OFulltext.FtsInput(ftq),
q.fix.account,
settings.batch
)
.map(Items.ftsItemsFull(true))
} else if (settings.resolveDetails) {
logger.debug(
s"Using index+sql search with tags: $validItemQuery / $fulltextQuery"
)
fts
.findItemsWithTags(settings.maxNoteLen)(
validItemQuery,
OFulltext.FtsInput(ftq),
settings.batch
)
.map(Items.ftsItemsFull(false))
} else {
logger.debug(
s"Using index+sql search no tags: $validItemQuery / $fulltextQuery"
)
fts
.findItems(settings.maxNoteLen)(
validItemQuery,
OFulltext.FtsInput(ftq),
settings.batch
)
.map(Items.ftsItems(false))
}
case _ =>
if (settings.resolveDetails) {
logger.debug(
s"Using sql only search with tags: $validItemQuery / $fulltextQuery"
)
is.findItemsWithTags(settings.maxNoteLen)(validItemQuery, settings.batch)
.map(Items.itemsFull)
} else {
logger.debug(
s"Using sql only search no tags: $validItemQuery / $fulltextQuery"
)
is.findItems(settings.maxNoteLen)(validItemQuery, settings.batch)
.map(Items.itemsPlain)
}
}
}
def searchSummary(
settings: StatsSettings
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = {
val validItemQuery =
settings.searchMode match {
case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed))
case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates))
case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed))
}
fulltextQuery match {
case Some(ftq) if settings.useFTS =>
if (q.isEmpty)
fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq))
else
fts
.findItemsSummary(validItemQuery, OFulltext.FtsInput(ftq))
case _ =>
is.findItemsSummary(validItemQuery)
}
}
}
}

45
modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala
View File

@ -8,13 +8,13 @@ package docspell.backend.ops.search
import java.time.LocalDate
import cats.data.OptionT
import cats.effect._
import cats.syntax.all._
import cats.{Functor, ~>}
import fs2.Stream
import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary}
import docspell.common.{AccountId, Duration, SearchMode}
import docspell.common._
import docspell.ftsclient.{FtsClient, FtsQuery}
import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser}
import docspell.store.Store
@ -35,7 +35,7 @@ trait OSearch[F[_]] {
* from fulltext search. Any "fulltext search" query node is discarded. It is assumed
* that the fulltext search node has been extracted into the argument.
*/
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)(
q: Query,
fulltextQuery: Option[String]
): F[Vector[ListItem]]
@ -45,7 +45,7 @@ trait OSearch[F[_]] {
*/
def searchWithDetails(
maxNoteLen: Int,
today: LocalDate,
today: Option[LocalDate],
batch: Batch
)(
q: Query,
@ -58,7 +58,7 @@ trait OSearch[F[_]] {
final def searchSelect(
withDetails: Boolean,
maxNoteLen: Int,
today: LocalDate,
today: Option[LocalDate],
batch: Batch
)(
q: Query,
@ -69,12 +69,14 @@ trait OSearch[F[_]] {
/** Run multiple database calls with the give query to collect a summary. */
def searchSummary(
today: LocalDate
today: Option[LocalDate]
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
/** Parses a query string and creates a `Query` object, to be used with the other
* methods. The query object contains the parsed query amended with more conditions to
* restrict to valid items only (as specified with `mode`).
* methods. The query object contains the parsed query amended with more conditions,
* for example to restrict to valid items only (as specified with `mode`). An empty
* query string is allowed and returns a query containing only the restrictions in the
* `q.fix` part.
*/
def parseQueryString(
accountId: AccountId,
@ -139,7 +141,7 @@ object OSearch {
}
}
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)(
q: Query,
fulltextQuery: Option[String]
): F[Vector[ListItem]] =
@ -148,6 +150,9 @@ object OSearch {
for {
timed <- Duration.stopTime[F]
ftq <- createFtsQuery(q.fix.account, ftq)
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
val tempTable = temporaryFtsTable(ftq, nat)
@ -156,7 +161,7 @@ object OSearch {
Stream
.eval(tempTable)
.flatMap(tt =>
QItem.queryItems(q, today, maxNoteLen, batch, tt.some)
QItem.queryItems(q, date, maxNoteLen, batch, tt.some)
)
)
.compile
@ -169,19 +174,21 @@ object OSearch {
case None =>
for {
timed <- Duration.stopTime[F]
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- store
.transact(QItem.queryItems(q, today, maxNoteLen, batch, None))
.transact(QItem.queryItems(q, date, maxNoteLen, batch, None))
.compile
.toVector
duration <- timed
_ <- logger.debug(s"Simple search sql in: ${duration.formatExact}")
} yield results
}
def searchWithDetails(
maxNoteLen: Int,
today: LocalDate,
today: Option[LocalDate],
batch: Batch
)(
q: Query,
@ -201,22 +208,28 @@ object OSearch {
} yield resolved
def searchSummary(
today: LocalDate
today: Option[LocalDate]
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
fulltextQuery match {
case Some(ftq) =>
for {
ftq <- createFtsQuery(q.fix.account, ftq)
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
val tempTable = temporaryFtsTable(ftq, nat)
store.transact(
tempTable.flatMap(tt => QItem.searchStats(today, tt.some)(q))
tempTable.flatMap(tt => QItem.searchStats(date, tt.some)(q))
)
}
} yield results
case None =>
store.transact(QItem.searchStats(today, None)(q))
OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
.flatMap(date => store.transact(QItem.searchStats(date, None)(q)))
}
private def createFtsQuery(

9
modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala
View File

@ -15,6 +15,8 @@ sealed trait QueryParseResult {
def get: Option[(Query, Option[String])]
def isSuccess: Boolean = get.isDefined
def isFailure: Boolean = !isSuccess
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult
}
object QueryParseResult {
@ -25,15 +27,22 @@ object QueryParseResult {
def withFtsEnabled(enabled: Boolean) =
if (enabled || ftq.isEmpty) this else copy(ftq = None)
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
f(this)
val get = Some(q -> ftq)
}
final case class ParseFailed(error: ParseFailure) extends QueryParseResult {
val get = None
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
this
}
final case class FulltextMismatch(error: FulltextExtract.FailureResult)
extends QueryParseResult {
val get = None
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
this
}
}