TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-05 19:09:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1581 from eikek/cleanup/search

Browse Source 
Remove unused code (search update)
This commit is contained in:
mergify[bot] 2022-06-05 21:48:08 +00:00 committed by GitHub
commit 2dcb5e266f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 228 additions and 1157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
modules/backend/src/main/scala/docspell/backend/BackendApp.scala
View File

@ -47,7 +47,6 @@ trait BackendApp[F[_]] {
def userTask: OUserTask[F] def userTask: OUserTask[F]
def folder: OFolder[F] def folder: OFolder[F]
def customFields: OCustomFields[F] def customFields: OCustomFields[F]
def simpleSearch: OSimpleSearch[F]
def clientSettings: OClientSettings[F] def clientSettings: OClientSettings[F]
def totp: OTotp[F] def totp: OTotp[F]
def share: OShare[F] def share: OShare[F]
@ -99,8 +98,6 @@ object BackendApp {
itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs) itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs)
itemSearchImpl <- OItemSearch(store) itemSearchImpl <- OItemSearch(store)
fulltextImpl <- OFulltext( fulltextImpl <- OFulltext(
itemSearchImpl,
ftsClient,
store, store,
schedulerModule.jobs schedulerModule.jobs
) )
@ -112,15 +109,15 @@ object BackendApp {
) )
folderImpl <- OFolder(store) folderImpl <- OFolder(store)
customFieldsImpl <- OCustomFields(store) customFieldsImpl <- OCustomFields(store)
simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl)
clientSettingsImpl <- OClientSettings(store) clientSettingsImpl <- OClientSettings(store)
searchImpl <- Resource.pure(OSearch(store, ftsClient))
shareImpl <- Resource.pure( shareImpl <- Resource.pure(
OShare(store, itemSearchImpl, simpleSearchImpl, javaEmil) OShare(store, itemSearchImpl, searchImpl, javaEmil)
) )
notifyImpl <- ONotification(store, notificationMod) notifyImpl <- ONotification(store, notificationMod)
bookmarksImpl <- OQueryBookmarks(store) bookmarksImpl <- OQueryBookmarks(store)
fileRepoImpl <- OFileRepository(store, schedulerModule.jobs) fileRepoImpl <- OFileRepository(store, schedulerModule.jobs)
itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl)) itemLinkImpl <- Resource.pure(OItemLink(store, searchImpl))
downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs)) downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs))
attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs)) attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs))
addonsImpl <- Resource.pure( addonsImpl <- Resource.pure(
@ -132,7 +129,6 @@ object BackendApp {
joexImpl joexImpl
) )
) )
searchImpl <- Resource.pure(OSearch(store, ftsClient))
} yield new BackendApp[F] { } yield new BackendApp[F] {
val pubSub = pubSubT val pubSub = pubSubT
val login = loginImpl val login = loginImpl
@ -153,7 +149,6 @@ object BackendApp {
val userTask = userTaskImpl val userTask = userTaskImpl
val folder = folderImpl val folder = folderImpl
val customFields = customFieldsImpl val customFields = customFieldsImpl
val simpleSearch = simpleSearchImpl
val clientSettings = clientSettingsImpl val clientSettings = clientSettingsImpl
val totp = totpImpl val totp = totpImpl
val share = shareImpl val share = shareImpl

281
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@ -6,46 +6,17 @@
package docspell.backend.ops package docspell.backend.ops
import cats.data.NonEmptyList
import cats.effect._ import cats.effect._
import cats.implicits._ import cats.implicits._
import fs2.Stream
import docspell.backend.JobFactory import docspell.backend.JobFactory
import docspell.backend.ops.OItemSearch._
import docspell.common._ import docspell.common._
import docspell.ftsclient._
import docspell.query.ItemQuery._
import docspell.query.ItemQueryDsl._
import docspell.scheduler.JobStore import docspell.scheduler.JobStore
import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.Store
import docspell.store.records.RJob import docspell.store.records.RJob
import docspell.store.{Store, qb}
trait OFulltext[F[_]] { trait OFulltext[F[_]] {
def findItems(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: qb.Batch
): F[Vector[OFulltext.FtsItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnly(maxNoteLen: Int)(
fts: OFulltext.FtsInput,
account: AccountId,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
/** Clears the full-text index completely and launches a task that indexes all data. */ /** Clears the full-text index completely and launches a task that indexes all data. */
def reindexAll: F[Unit] def reindexAll: F[Unit]
@ -56,30 +27,7 @@ trait OFulltext[F[_]] {
} }
object OFulltext { object OFulltext {
case class FtsInput(
query: String,
highlightPre: String = "***",
highlightPost: String = "***"
)
case class FtsDataItem(
score: Double,
matchData: FtsResult.MatchData,
context: List[String]
)
case class FtsData(
maxScore: Double,
count: Int,
qtime: Duration,
items: List[FtsDataItem]
)
case class FtsItem(item: ListItem, ftsData: FtsData)
case class FtsItemWithTags(item: ListItemWithTags, ftsData: FtsData)
def apply[F[_]: Async]( def apply[F[_]: Async](
itemSearch: OItemSearch[F],
fts: FtsClient[F],
store: Store[F], store: Store[F],
jobStore: JobStore[F] jobStore: JobStore[F]
): Resource[F, OFulltext[F]] = ): Resource[F, OFulltext[F]] =
@ -103,232 +51,5 @@ object OFulltext {
if (exist.isDefined) ().pure[F] if (exist.isDefined) ().pure[F]
else jobStore.insertIfNew(job.encode) else jobStore.insertIfNew(job.encode)
} yield () } yield ()
def findIndexOnly(maxNoteLen: Int)(
ftsQ: OFulltext.FtsInput,
account: AccountId,
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]] = {
val fq = FtsQuery(
ftsQ.query,
account.collective,
Set.empty,
Set.empty,
batch.limit,
batch.offset,
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
)
for {
_ <- logger.trace(s"Find index only: ${ftsQ.query}/$batch")
folders <- store.transact(QFolder.getMemberFolders(account))
ftsR <- fts.search(fq.withFolders(folders))
ftsItems = ftsR.results.groupBy(_.itemId)
select =
ftsItems.values
.map(_.minBy(-_.score))
.map(r => SelectedItem(r.itemId, r.score))
.toSet
now <- Timestamp.current[F]
itemsWithTags <-
store
.transact(
QItem.findItemsWithTags(
account.collective,
QItem.findSelectedItems(
Query.all(account),
now.toUtcDate,
maxNoteLen,
select
)
)
)
.take(batch.limit.toLong)
.compile
.toVector
res =
itemsWithTags
.collect(convertFtsData(ftsR, ftsItems))
.map { case (li, fd) => FtsItemWithTags(li, fd) }
} yield res
}
def findIndexOnlySummary(
account: AccountId,
ftsQ: OFulltext.FtsInput
): F[SearchSummary] = {
val fq = FtsQuery(
ftsQ.query,
account.collective,
Set.empty,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
for {
folder <- store.transact(QFolder.getMemberFolders(account))
now <- Timestamp.current[F]
itemIds <- fts
.searchAll(fq.withFolders(folder))
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
itemIdsQuery = NonEmptyList
.fromList(itemIds.toList)
.map(ids => Attr.ItemId.in(ids.map(_.id)))
.getOrElse(Attr.ItemId.notExists)
q = Query
.all(account)
.withFix(_.copy(query = itemIdsQuery.some))
res <- store.transact(QItem.searchStats(now.toUtcDate, None)(q))
} yield res
}
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItems(maxNoteLen),
convertFtsData[ListItem]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map { case (li, fd) => FtsItem(li, fd) }
.compile
.toVector
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
ftsQ: FtsInput,
batch: qb.Batch
): F[Vector[FtsItemWithTags]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItemsWithTags(maxNoteLen),
convertFtsData[ListItemWithTags]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map { case (li, fd) => FtsItemWithTags(li, fd) }
.compile
.toVector
def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] =
for {
search <- itemSearch.findItems(0)(q, Batch.all)
fq = FtsQuery(
ftsQ.query,
q.fix.account.collective,
search.map(_.id).toSet,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
items <- fts
.searchAll(fq)
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
itemIdsQuery = NonEmptyList
.fromList(items.toList)
.map(ids => Attr.ItemId.in(ids.map(_.id)))
.getOrElse(Attr.ItemId.notExists)
qnext = q.withFix(_.copy(query = itemIdsQuery.some))
now <- Timestamp.current[F]
res <- store.transact(QItem.searchStats(now.toUtcDate, None)(qnext))
} yield res
// Helper
private def findItemsFts[A: ItemId, B](
q: Query,
ftsQ: FtsInput,
batch: qb.Batch,
search: (Query, qb.Batch) => F[Vector[A]],
convert: (
FtsResult,
Map[Ident, List[FtsResult.ItemMatch]]
) => PartialFunction[A, (A, FtsData)]
): Stream[F, (A, FtsData)] =
findItemsFts0(q, ftsQ, batch, search, convert)
.takeThrough(_._1 >= batch.limit)
.flatMap(x => Stream.emits(x._2))
private def findItemsFts0[A: ItemId, B](
q: Query,
ftsQ: FtsInput,
batch: qb.Batch,
search: (Query, qb.Batch) => F[Vector[A]],
convert: (
FtsResult,
Map[Ident, List[FtsResult.ItemMatch]]
) => PartialFunction[A, (A, FtsData)]
): Stream[F, (Int, Vector[(A, FtsData)])] = {
val sqlResult = search(q, batch)
val fq = FtsQuery(
ftsQ.query,
q.fix.account.collective,
Set.empty,
Set.empty,
0,
0,
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
)
val qres =
for {
items <- sqlResult
ids = items.map(a => ItemId[A].itemId(a))
idsNel = NonEmptyList.fromFoldable(ids)
// must find all index results involving the items.
// Currently there is one result per item + one result per
// attachment
limit <- idsNel
.map(itemIds => store.transact(QItem.countAttachmentsAndItems(itemIds)))
.getOrElse(0.pure[F])
ftsQ = fq.copy(items = ids.toSet, limit = limit)
ftsR <- fts.search(ftsQ)
ftsItems = ftsR.results.groupBy(_.itemId)
res = items.collect(convert(ftsR, ftsItems))
} yield (items.size, res)
Stream.eval(qres) ++ findItemsFts0(q, ftsQ, batch.next, search, convert)
}
private def convertFtsData[A: ItemId](
ftr: FtsResult,
ftrItems: Map[Ident, List[FtsResult.ItemMatch]]
): PartialFunction[A, (A, FtsData)] = {
case a if ftrItems.contains(ItemId[A].itemId(a)) =>
val ftsDataItems = ftrItems
.getOrElse(ItemId[A].itemId(a), Nil)
.map(im =>
FtsDataItem(im.score, im.data, ftr.highlight.getOrElse(im.id, Nil))
)
(a, FtsData(ftr.maxScore, ftr.count, ftr.qtime, ftsDataItems))
}
}) })
trait ItemId[A] {
def itemId(a: A): Ident
}
object ItemId {
def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev
def from[A](f: A => Ident): ItemId[A] =
(a: A) => f(a)
implicit val listItemId: ItemId[ListItem] =
ItemId.from(_.id)
implicit val listItemWithTagsId: ItemId[ListItemWithTags] =
ItemId.from(_.item.id)
}
} }

13
modules/backend/src/main/scala/docspell/backend/ops/OItemLink.scala
View File

@ -11,11 +11,12 @@ import cats.effect._
import cats.implicits._ import cats.implicits._
import docspell.backend.ops.OItemLink.LinkResult import docspell.backend.ops.OItemLink.LinkResult
import docspell.backend.ops.search.OSearch
import docspell.common.{AccountId, Ident} import docspell.common.{AccountId, Ident}
import docspell.query.ItemQuery import docspell.query.ItemQuery
import docspell.query.ItemQueryDsl._ import docspell.query.ItemQueryDsl._
import docspell.store.qb.Batch import docspell.store.qb.Batch
import docspell.store.queries.Query import docspell.store.queries.{ListItemWithTags, Query}
import docspell.store.records.RItemLink import docspell.store.records.RItemLink
import docspell.store.{AddResult, Store} import docspell.store.{AddResult, Store}
@ -29,7 +30,7 @@ trait OItemLink[F[_]] {
account: AccountId, account: AccountId,
item: Ident, item: Ident,
batch: Batch batch: Batch
): F[Vector[OItemSearch.ListItemWithTags]] ): F[Vector[ListItemWithTags]]
} }
object OItemLink { object OItemLink {
@ -44,13 +45,13 @@ object OItemLink {
def linkTargetItemError: LinkResult = LinkTargetItemError def linkTargetItemError: LinkResult = LinkTargetItemError
} }
def apply[F[_]: Sync](store: Store[F], search: OItemSearch[F]): OItemLink[F] = def apply[F[_]: Sync](store: Store[F], search: OSearch[F]): OItemLink[F] =
new OItemLink[F] { new OItemLink[F] {
def getRelated( def getRelated(
accountId: AccountId, accountId: AccountId,
item: Ident, item: Ident,
batch: Batch batch: Batch
): F[Vector[OItemSearch.ListItemWithTags]] = ): F[Vector[ListItemWithTags]] =
store store
.transact(RItemLink.findLinked(accountId.collective, item)) .transact(RItemLink.findLinked(accountId.collective, item))
.map(ids => NonEmptyList.fromList(ids.toList)) .map(ids => NonEmptyList.fromList(ids.toList))
@ -62,10 +63,10 @@ object OItemLink {
.Fix(accountId, Some(ItemQuery.Expr.ValidItemStates), None), .Fix(accountId, Some(ItemQuery.Expr.ValidItemStates), None),
Query.QueryExpr(expr) Query.QueryExpr(expr)
) )
search.findItemsWithTags(0)(query, batch) search.searchWithDetails(0, None, batch)(query, None)
case None => case None =>
Vector.empty[OItemSearch.ListItemWithTags].pure[F] Vector.empty[ListItemWithTags].pure[F]
} }
def addAll(cid: Ident, target: Ident, related: NonEmptyList[Ident]): F[LinkResult] = def addAll(cid: Ident, target: Ident, related: NonEmptyList[Ident]): F[LinkResult] =

54
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@ -25,15 +25,6 @@ trait OItemSearch[F[_]] {
def findDeleted(collective: Ident, maxUpdate: Timestamp, limit: Int): F[Vector[RItem]] def findDeleted(collective: Ident, maxUpdate: Timestamp, limit: Int): F[Vector[RItem]]
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findItemsSummary(q: Query): F[SearchSummary]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
def findAttachmentSource( def findAttachmentSource(
@ -63,9 +54,6 @@ trait OItemSearch[F[_]] {
object OItemSearch { object OItemSearch {
type SearchSummary = queries.SearchSummary
val SearchSummary = queries.SearchSummary
type CustomValue = queries.CustomValue type CustomValue = queries.CustomValue
val CustomValue = queries.CustomValue val CustomValue = queries.CustomValue
@ -75,12 +63,6 @@ object OItemSearch {
type Batch = qb.Batch type Batch = qb.Batch
val Batch = docspell.store.qb.Batch val Batch = docspell.store.qb.Batch
type ListItem = queries.ListItem
val ListItem = queries.ListItem
type ListItemWithTags = queries.ListItemWithTags
val ListItemWithTags = queries.ListItemWithTags
type ItemFieldValue = queries.ItemFieldValue type ItemFieldValue = queries.ItemFieldValue
val ItemFieldValue = queries.ItemFieldValue val ItemFieldValue = queries.ItemFieldValue
@ -136,19 +118,6 @@ object OItemSearch {
store store
.transact(QItem.findItem(id, collective)) .transact(QItem.findItem(id, collective))
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap { today =>
store
.transact(
QItem.findItems(q, today, maxNoteLen, batch).take(batch.limit.toLong)
)
.compile
.toVector
}
def findDeleted( def findDeleted(
collective: Ident, collective: Ident,
maxUpdate: Timestamp, maxUpdate: Timestamp,
@ -160,28 +129,6 @@ object OItemSearch {
.compile .compile
.toVector .toVector
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
for {
now <- Timestamp.current[F]
search = QItem.findItems(q, now.toUtcDate, maxNoteLen: Int, batch)
res <- store
.transact(
QItem
.findItemsWithTags(q.fix.account.collective, search)
.take(batch.limit.toLong)
)
.compile
.toVector
} yield res
def findItemsSummary(q: Query): F[SearchSummary] =
Timestamp
.current[F]
.map(_.toUtcDate)
.flatMap(today => store.transact(QItem.searchStats(today, None)(q)))
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store store
.transact(RAttachment.findByIdAndCollective(id, collective)) .transact(RAttachment.findByIdAndCollective(id, collective))
@ -298,6 +245,5 @@ object OItemSearch {
coll <- OptionT(RSource.findCollective(sourceId)) coll <- OptionT(RSource.findCollective(sourceId))
items <- OptionT.liftF(QItem.findByChecksum(checksum, coll, Set.empty)) items <- OptionT.liftF(QItem.findByChecksum(checksum, coll, Set.empty))
} yield items).value) } yield items).value)
}) })
} }

37
modules/backend/src/main/scala/docspell/backend/ops/OShare.scala
View File

@ -14,13 +14,12 @@ import docspell.backend.PasswordCrypt
import docspell.backend.auth.ShareToken import docspell.backend.auth.ShareToken
import docspell.backend.ops.OItemSearch._ import docspell.backend.ops.OItemSearch._
import docspell.backend.ops.OShare._ import docspell.backend.ops.OShare._
import docspell.backend.ops.OSimpleSearch.StringSearchResult import docspell.backend.ops.search.{OSearch, QueryParseResult}
import docspell.common._ import docspell.common._
import docspell.query.ItemQuery.Expr import docspell.query.ItemQuery.Expr
import docspell.query.ItemQuery.Expr.AttachId import docspell.query.ItemQuery.Expr.AttachId
import docspell.query.{FulltextExtract, ItemQuery} import docspell.query.{FulltextExtract, ItemQuery}
import docspell.store.Store import docspell.store.Store
import docspell.store.queries.SearchSummary
import docspell.store.records._ import docspell.store.records._
import emil._ import emil._
@ -67,9 +66,10 @@ trait OShare[F[_]] {
def findItem(itemId: Ident, shareId: Ident): OptionT[F, ItemData] def findItem(itemId: Ident, shareId: Ident): OptionT[F, ItemData]
def searchSummary( /** Parses a query and amends the result with the stored query of the share. The result
settings: OSimpleSearch.StatsSettings * can be used with [[OSearch]] to search for items.
)(shareId: Ident, q: ItemQueryString): OptionT[F, StringSearchResult[SearchSummary]] */
def parseQuery(share: ShareQuery, qs: String): QueryParseResult
def sendMail(account: AccountId, connection: Ident, mail: ShareMail): F[SendResult] def sendMail(account: AccountId, connection: Ident, mail: ShareMail): F[SendResult]
} }
@ -148,7 +148,7 @@ object OShare {
def apply[F[_]: Async]( def apply[F[_]: Async](
store: Store[F], store: Store[F],
itemSearch: OItemSearch[F], itemSearch: OItemSearch[F],
simpleSearch: OSimpleSearch[F], search: OSearch[F],
emil: Emil[F] emil: Emil[F]
): OShare[F] = ): OShare[F] =
new OShare[F] { new OShare[F] {
@ -325,8 +325,8 @@ object OShare {
Query.QueryExpr(idExpr) Query.QueryExpr(idExpr)
) )
OptionT( OptionT(
itemSearch search
.findItems(0)(checkQuery, Batch.limit(1)) .search(0, None, Batch.limit(1))(checkQuery, None)
.map(_.headOption.map(_ => ())) .map(_.headOption.map(_ => ()))
).flatTapNone( ).flatTapNone(
logger.info( logger.info(
@ -335,22 +335,11 @@ object OShare {
) )
} }
def searchSummary( def parseQuery(share: ShareQuery, qs: String): QueryParseResult =
settings: OSimpleSearch.StatsSettings search
)( .parseQueryString(share.account, SearchMode.Normal, qs)
shareId: Ident, .map { case QueryParseResult.Success(q, ftq) =>
q: ItemQueryString QueryParseResult.Success(q.withFix(_.andQuery(share.query.expr)), ftq)
): OptionT[F, StringSearchResult[SearchSummary]] =
findShareQuery(shareId)
.semiflatMap { share =>
val fix = Query.Fix(share.account, Some(share.query.expr), None)
simpleSearch
.searchSummaryByString(settings)(fix, q)
.map {
case StringSearchResult.Success(summary) =>
StringSearchResult.Success(summary.onlyExisting)
case other => other
}
} }
def sendMail( def sendMail(

300
modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala
View File

@ -1,300 +0,0 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.backend.ops
import cats.Applicative
import cats.effect.Sync
import cats.implicits._
import docspell.backend.ops.OSimpleSearch._
import docspell.common._
import docspell.query._
import docspell.store.qb.Batch
import docspell.store.queries.Query
import docspell.store.queries.SearchSummary
import org.log4s.getLogger
/** A "porcelain" api on top of OFulltext and OItemSearch. This takes care of restricting
* the items to a subset, e.g. only items that have a "valid" state.
*/
trait OSimpleSearch[F[_]] {
/** Search for items using the given query and optional fulltext search.
*
* When using fulltext search only (the query is empty), only the index is searched. It
* is assumed that the index doesn't contain "invalid" items. When using a query, then
* a condition to select only valid items is added to it.
*/
def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
/** Using the same arguments as in `search`, this returns a summary and not the results.
*/
def searchSummary(
settings: StatsSettings
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
/** Calls `search` by parsing the given query string into a query that is then amended
* wtih the given `fix` query.
*/
final def searchByString(
settings: Settings
)(fix: Query.Fix, q: ItemQueryString)(implicit
F: Applicative[F]
): F[StringSearchResult[Items]] =
OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts))
/** Same as `searchByString` but returning a summary instead of the results. */
final def searchSummaryByString(
settings: StatsSettings
)(fix: Query.Fix, q: ItemQueryString)(implicit
F: Applicative[F]
): F[StringSearchResult[SearchSummary]] =
OSimpleSearch.applySearch[F, SearchSummary](fix, q)((iq, fts) =>
searchSummary(settings)(iq, fts)
)
}
object OSimpleSearch {
private[this] val logger = getLogger
sealed trait StringSearchResult[+A]
object StringSearchResult {
case class ParseFailed(error: ParseFailure) extends StringSearchResult[Nothing]
def parseFailed[A](error: ParseFailure): StringSearchResult[A] =
ParseFailed(error)
case class FulltextMismatch(error: FulltextExtract.FailureResult)
extends StringSearchResult[Nothing]
def fulltextMismatch[A](error: FulltextExtract.FailureResult): StringSearchResult[A] =
FulltextMismatch(error)
case class Success[A](value: A) extends StringSearchResult[A]
}
final case class Settings(
batch: Batch,
useFTS: Boolean,
resolveDetails: Boolean,
maxNoteLen: Int,
searchMode: SearchMode
)
final case class StatsSettings(
useFTS: Boolean,
searchMode: SearchMode
)
sealed trait Items {
def fold[A](
f1: Items.FtsItems => A,
f2: Items.FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A
}
object Items {
def ftsItems(indexOnly: Boolean)(items: Vector[OFulltext.FtsItem]): Items =
FtsItems(items, indexOnly)
case class FtsItems(items: Vector[OFulltext.FtsItem], indexOnly: Boolean)
extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f1(this)
}
def ftsItemsFull(indexOnly: Boolean)(
items: Vector[OFulltext.FtsItemWithTags]
): Items =
FtsItemsFull(items, indexOnly)
case class FtsItemsFull(items: Vector[OFulltext.FtsItemWithTags], indexOnly: Boolean)
extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f2(this)
}
def itemsPlain(items: Vector[OItemSearch.ListItem]): Items =
ItemsPlain(items)
case class ItemsPlain(items: Vector[OItemSearch.ListItem]) extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f3(items)
}
def itemsFull(items: Vector[OItemSearch.ListItemWithTags]): Items =
ItemsFull(items)
case class ItemsFull(items: Vector[OItemSearch.ListItemWithTags]) extends Items {
def fold[A](
f1: FtsItems => A,
f2: FtsItemsFull => A,
f3: Vector[OItemSearch.ListItem] => A,
f4: Vector[OItemSearch.ListItemWithTags] => A
): A = f4(items)
}
}
def apply[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]): OSimpleSearch[F] =
new Impl(fts, is)
/** Parses the query and calls `run` with the result, which searches items. */
private def applySearch[F[_]: Applicative, A](fix: Query.Fix, q: ItemQueryString)(
run: (Query, Option[String]) => F[A]
): F[StringSearchResult[A]] = {
val parsed: Either[StringSearchResult[A], Option[ItemQuery]] =
if (q.isEmpty) Right(None)
else
ItemQueryParser
.parse(q.query)
.leftMap(StringSearchResult.parseFailed)
.map(_.some)
def makeQuery(itemQuery: Option[ItemQuery]): F[StringSearchResult[A]] =
runQuery[F, A](itemQuery) {
case Some(s) =>
run(Query(fix, Query.QueryExpr(s.getExprPart)), s.getFulltextPart)
case None =>
run(Query(fix), None)
}
parsed match {
case Right(iq) =>
makeQuery(iq)
case Left(err) =>
err.pure[F]
}
}
/** Calls `run` with one of the success results when extracting the fulltext search node
* from the query.
*/
private def runQuery[F[_]: Applicative, A](
itemQuery: Option[ItemQuery]
)(run: Option[FulltextExtract.SuccessResult] => F[A]): F[StringSearchResult[A]] =
itemQuery match {
case Some(iq) =>
iq.findFulltext match {
case s: FulltextExtract.SuccessResult =>
run(Some(s)).map(StringSearchResult.Success.apply)
case other: FulltextExtract.FailureResult =>
StringSearchResult.fulltextMismatch[A](other).pure[F]
}
case None =>
run(None).map(StringSearchResult.Success.apply)
}
final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F])
extends OSimpleSearch[F] {
/** Implements searching like this: it exploits the fact that teh fulltext index only
* contains valid items. When searching via sql the query expression selecting only
* valid items is added here.
*/
def search(
settings: Settings
)(q: Query, fulltextQuery: Option[String]): F[Items] = {
// 1. fulltext only if fulltextQuery.isDefined && q.isEmpty && useFTS
// 2. sql+fulltext if fulltextQuery.isDefined && q.nonEmpty && useFTS
// 3. sql-only else (if fulltextQuery.isEmpty || !useFTS)
val validItemQuery =
settings.searchMode match {
case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed))
case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates))
case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed))
}
fulltextQuery match {
case Some(ftq) if settings.useFTS =>
if (q.isEmpty) {
logger.debug(s"Using index only search: $fulltextQuery")
if (settings.searchMode == SearchMode.Trashed)
Items.ftsItemsFull(true)(Vector.empty).pure[F]
else
fts
.findIndexOnly(settings.maxNoteLen)(
OFulltext.FtsInput(ftq),
q.fix.account,
settings.batch
)
.map(Items.ftsItemsFull(true))
} else if (settings.resolveDetails) {
logger.debug(
s"Using index+sql search with tags: $validItemQuery / $fulltextQuery"
)
fts
.findItemsWithTags(settings.maxNoteLen)(
validItemQuery,
OFulltext.FtsInput(ftq),
settings.batch
)
.map(Items.ftsItemsFull(false))
} else {
logger.debug(
s"Using index+sql search no tags: $validItemQuery / $fulltextQuery"
)
fts
.findItems(settings.maxNoteLen)(
validItemQuery,
OFulltext.FtsInput(ftq),
settings.batch
)
.map(Items.ftsItems(false))
}
case _ =>
if (settings.resolveDetails) {
logger.debug(
s"Using sql only search with tags: $validItemQuery / $fulltextQuery"
)
is.findItemsWithTags(settings.maxNoteLen)(validItemQuery, settings.batch)
.map(Items.itemsFull)
} else {
logger.debug(
s"Using sql only search no tags: $validItemQuery / $fulltextQuery"
)
is.findItems(settings.maxNoteLen)(validItemQuery, settings.batch)
.map(Items.itemsPlain)
}
}
}
def searchSummary(
settings: StatsSettings
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = {
val validItemQuery =
settings.searchMode match {
case SearchMode.Trashed => q.withFix(_.andQuery(ItemQuery.Expr.Trashed))
case SearchMode.Normal => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemStates))
case SearchMode.All => q.withFix(_.andQuery(ItemQuery.Expr.ValidItemsOrTrashed))
}
fulltextQuery match {
case Some(ftq) if settings.useFTS =>
if (q.isEmpty)
fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq))
else
fts
.findItemsSummary(validItemQuery, OFulltext.FtsInput(ftq))
case _ =>
is.findItemsSummary(validItemQuery)
}
}
}
}

45
modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala
View File

@ -8,13 +8,13 @@ package docspell.backend.ops.search
import java.time.LocalDate import java.time.LocalDate
import cats.data.OptionT
import cats.effect._ import cats.effect._
import cats.syntax.all._ import cats.syntax.all._
import cats.{Functor, ~>} import cats.{Functor, ~>}
import fs2.Stream import fs2.Stream
import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary} import docspell.common._
import docspell.common.{AccountId, Duration, SearchMode}
import docspell.ftsclient.{FtsClient, FtsQuery} import docspell.ftsclient.{FtsClient, FtsQuery}
import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser} import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser}
import docspell.store.Store import docspell.store.Store
@ -35,7 +35,7 @@ trait OSearch[F[_]] {
* from fulltext search. Any "fulltext search" query node is discarded. It is assumed * from fulltext search. Any "fulltext search" query node is discarded. It is assumed
* that the fulltext search node has been extracted into the argument. * that the fulltext search node has been extracted into the argument.
*/ */
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)( def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)(
q: Query, q: Query,
fulltextQuery: Option[String] fulltextQuery: Option[String]
): F[Vector[ListItem]] ): F[Vector[ListItem]]
@ -45,7 +45,7 @@ trait OSearch[F[_]] {
*/ */
def searchWithDetails( def searchWithDetails(
maxNoteLen: Int, maxNoteLen: Int,
today: LocalDate, today: Option[LocalDate],
batch: Batch batch: Batch
)( )(
q: Query, q: Query,
@ -58,7 +58,7 @@ trait OSearch[F[_]] {
final def searchSelect( final def searchSelect(
withDetails: Boolean, withDetails: Boolean,
maxNoteLen: Int, maxNoteLen: Int,
today: LocalDate, today: Option[LocalDate],
batch: Batch batch: Batch
)( )(
q: Query, q: Query,
@ -69,12 +69,14 @@ trait OSearch[F[_]] {
/** Run multiple database calls with the give query to collect a summary. */ /** Run multiple database calls with the give query to collect a summary. */
def searchSummary( def searchSummary(
today: LocalDate today: Option[LocalDate]
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
/** Parses a query string and creates a `Query` object, to be used with the other /** Parses a query string and creates a `Query` object, to be used with the other
* methods. The query object contains the parsed query amended with more conditions to * methods. The query object contains the parsed query amended with more conditions,
* restrict to valid items only (as specified with `mode`). * for example to restrict to valid items only (as specified with `mode`). An empty
* query string is allowed and returns a query containing only the restrictions in the
* `q.fix` part.
*/ */
def parseQueryString( def parseQueryString(
accountId: AccountId, accountId: AccountId,
@ -139,7 +141,7 @@ object OSearch {
} }
} }
def search(maxNoteLen: Int, today: LocalDate, batch: Batch)( def search(maxNoteLen: Int, today: Option[LocalDate], batch: Batch)(
q: Query, q: Query,
fulltextQuery: Option[String] fulltextQuery: Option[String]
): F[Vector[ListItem]] = ): F[Vector[ListItem]] =
@ -148,6 +150,9 @@ object OSearch {
for { for {
timed <- Duration.stopTime[F] timed <- Duration.stopTime[F]
ftq <- createFtsQuery(q.fix.account, ftq) ftq <- createFtsQuery(q.fix.account, ftq)
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- WeakAsync.liftK[F, ConnectionIO].use { nat => results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
val tempTable = temporaryFtsTable(ftq, nat) val tempTable = temporaryFtsTable(ftq, nat)
@ -156,7 +161,7 @@ object OSearch {
Stream Stream
.eval(tempTable) .eval(tempTable)
.flatMap(tt => .flatMap(tt =>
QItem.queryItems(q, today, maxNoteLen, batch, tt.some) QItem.queryItems(q, date, maxNoteLen, batch, tt.some)
) )
) )
.compile .compile
@ -169,19 +174,21 @@ object OSearch {
case None => case None =>
for { for {
timed <- Duration.stopTime[F] timed <- Duration.stopTime[F]
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- store results <- store
.transact(QItem.queryItems(q, today, maxNoteLen, batch, None)) .transact(QItem.queryItems(q, date, maxNoteLen, batch, None))
.compile .compile
.toVector .toVector
duration <- timed duration <- timed
_ <- logger.debug(s"Simple search sql in: ${duration.formatExact}") _ <- logger.debug(s"Simple search sql in: ${duration.formatExact}")
} yield results } yield results
} }
def searchWithDetails( def searchWithDetails(
maxNoteLen: Int, maxNoteLen: Int,
today: LocalDate, today: Option[LocalDate],
batch: Batch batch: Batch
)( )(
q: Query, q: Query,
@ -201,22 +208,28 @@ object OSearch {
} yield resolved } yield resolved
def searchSummary( def searchSummary(
today: LocalDate today: Option[LocalDate]
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
fulltextQuery match { fulltextQuery match {
case Some(ftq) => case Some(ftq) =>
for { for {
ftq <- createFtsQuery(q.fix.account, ftq) ftq <- createFtsQuery(q.fix.account, ftq)
date <- OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
results <- WeakAsync.liftK[F, ConnectionIO].use { nat => results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
val tempTable = temporaryFtsTable(ftq, nat) val tempTable = temporaryFtsTable(ftq, nat)
store.transact( store.transact(
tempTable.flatMap(tt => QItem.searchStats(today, tt.some)(q)) tempTable.flatMap(tt => QItem.searchStats(date, tt.some)(q))
) )
} }
} yield results } yield results
case None => case None =>
store.transact(QItem.searchStats(today, None)(q)) OptionT
.fromOption(today)
.getOrElseF(Timestamp.current[F].map(_.toUtcDate))
.flatMap(date => store.transact(QItem.searchStats(date, None)(q)))
} }
private def createFtsQuery( private def createFtsQuery(

9
modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala
View File

@ -15,6 +15,8 @@ sealed trait QueryParseResult {
def get: Option[(Query, Option[String])] def get: Option[(Query, Option[String])]
def isSuccess: Boolean = get.isDefined def isSuccess: Boolean = get.isDefined
def isFailure: Boolean = !isSuccess def isFailure: Boolean = !isSuccess
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult
} }
object QueryParseResult { object QueryParseResult {
@ -25,15 +27,22 @@ object QueryParseResult {
def withFtsEnabled(enabled: Boolean) = def withFtsEnabled(enabled: Boolean) =
if (enabled || ftq.isEmpty) this else copy(ftq = None) if (enabled || ftq.isEmpty) this else copy(ftq = None)
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
f(this)
val get = Some(q -> ftq) val get = Some(q -> ftq)
} }
final case class ParseFailed(error: ParseFailure) extends QueryParseResult { final case class ParseFailed(error: ParseFailure) extends QueryParseResult {
val get = None val get = None
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
this
} }
final case class FulltextMismatch(error: FulltextExtract.FailureResult) final case class FulltextMismatch(error: FulltextExtract.FailureResult)
extends QueryParseResult { extends QueryParseResult {
val get = None val get = None
def map(f: QueryParseResult.Success => QueryParseResult): QueryParseResult =
this
} }
} }

6
modules/joex/src/main/scala/docspell/joex/JoexTasks.scala
View File

@ -13,6 +13,7 @@ import docspell.backend.BackendCommands
import docspell.backend.fulltext.CreateIndex import docspell.backend.fulltext.CreateIndex
import docspell.backend.joex.AddonOps import docspell.backend.joex.AddonOps
import docspell.backend.ops._ import docspell.backend.ops._
import docspell.backend.ops.search.OSearch
import docspell.backend.task.DownloadZipArgs import docspell.backend.task.DownloadZipArgs
import docspell.common._ import docspell.common._
import docspell.config.FtsType import docspell.config.FtsType
@ -62,6 +63,7 @@ final class JoexTasks[F[_]: Async](
joex: OJoex[F], joex: OJoex[F],
jobs: OJob[F], jobs: OJob[F],
itemSearch: OItemSearch[F], itemSearch: OItemSearch[F],
search: OSearch[F],
addons: AddonOps[F] addons: AddonOps[F]
) { ) {
val downloadAll: ODownloadAll[F] = val downloadAll: ODownloadAll[F] =
@ -201,7 +203,7 @@ final class JoexTasks[F[_]: Async](
.withTask( .withTask(
JobTask.json( JobTask.json(
PeriodicQueryTask.taskName, PeriodicQueryTask.taskName,
PeriodicQueryTask[F](store, notification), PeriodicQueryTask[F](store, search, notification),
PeriodicQueryTask.onCancel[F] PeriodicQueryTask.onCancel[F]
) )
) )
@ -273,6 +275,7 @@ object JoexTasks {
createIndex <- CreateIndex.resource(fts, store) createIndex <- CreateIndex.resource(fts, store)
itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs) itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs)
itemSearchOps <- OItemSearch(store) itemSearchOps <- OItemSearch(store)
searchOps <- Resource.pure(OSearch(store, fts))
analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig) analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig)
regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store)
updateCheck <- UpdateCheck.resource(httpClient) updateCheck <- UpdateCheck.resource(httpClient)
@ -306,6 +309,7 @@ object JoexTasks {
joex, joex,
jobs, jobs,
itemSearchOps, itemSearchOps,
searchOps,
addons addons
) )

2
modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala
View File

@ -89,7 +89,7 @@ object PeriodicDueItemsTask {
store store
.transact( .transact(
QItem QItem
.findItems(q, now.toUtcDate, 0, Batch.limit(limit)) .queryItems(q, now.toUtcDate, 0, Batch.limit(limit), None)
.take(limit.toLong) .take(limit.toLong)
) )
.compile .compile

76
modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala
View File

@ -7,25 +7,21 @@
package docspell.joex.notify package docspell.joex.notify
import cats.data.OptionT import cats.data.OptionT
import cats.data.{NonEmptyList => Nel}
import cats.effect._ import cats.effect._
import cats.implicits._ import cats.implicits._
import docspell.backend.ops.ONotification import docspell.backend.ops.ONotification
import docspell.backend.ops.search.{OSearch, QueryParseResult}
import docspell.common._ import docspell.common._
import docspell.notification.api.EventContext import docspell.notification.api.EventContext
import docspell.notification.api.NotificationChannel import docspell.notification.api.NotificationChannel
import docspell.notification.api.PeriodicQueryArgs import docspell.notification.api.PeriodicQueryArgs
import docspell.query.ItemQuery import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser}
import docspell.query.ItemQuery.Expr
import docspell.query.ItemQuery.Expr.AndExpr
import docspell.query.ItemQueryParser
import docspell.scheduler.Context import docspell.scheduler.Context
import docspell.scheduler.Task import docspell.scheduler.Task
import docspell.store.Store import docspell.store.Store
import docspell.store.qb.Batch import docspell.store.qb.Batch
import docspell.store.queries.ListItem import docspell.store.queries.ListItem
import docspell.store.queries.{QItem, Query}
import docspell.store.records.RQueryBookmark import docspell.store.records.RQueryBookmark
import docspell.store.records.RShare import docspell.store.records.RShare
@ -39,12 +35,13 @@ object PeriodicQueryTask {
def apply[F[_]: Sync]( def apply[F[_]: Sync](
store: Store[F], store: Store[F],
search: OSearch[F],
notificationOps: ONotification[F] notificationOps: ONotification[F]
): Task[F, Args, Unit] = ): Task[F, Args, Unit] =
Task { ctx => Task { ctx =>
val limit = 7 val limit = 7
Timestamp.current[F].flatMap { now => Timestamp.current[F].flatMap { now =>
withItems(ctx, store, limit, now) { items => withItems(ctx, store, search, limit, now) { items =>
withEventContext(ctx, items, limit, now) { eventCtx => withEventContext(ctx, items, limit, now) { eventCtx =>
withChannel(ctx, notificationOps) { channels => withChannel(ctx, notificationOps) { channels =>
notificationOps.sendMessage(ctx.logger, eventCtx, channels) notificationOps.sendMessage(ctx.logger, eventCtx, channels)
@ -62,8 +59,8 @@ object PeriodicQueryTask {
private def queryString(q: ItemQuery.Expr) = private def queryString(q: ItemQuery.Expr) =
ItemQueryParser.asString(q) ItemQueryParser.asString(q)
def withQuery[F[_]: Sync](ctx: Context[F, Args], store: Store[F])( def withQuery[F[_]: Sync](ctx: Context[F, Args], store: Store[F], search: OSearch[F])(
cont: Query => F[Unit] cont: QueryParseResult.Success => F[Unit]
): F[Unit] = { ): F[Unit] = {
def fromBookmark(id: String) = def fromBookmark(id: String) =
store store
@ -84,33 +81,51 @@ object PeriodicQueryTask {
def fromBookmarkOrShare(id: String) = def fromBookmarkOrShare(id: String) =
OptionT(fromBookmark(id)).orElse(OptionT(fromShare(id))).value OptionT(fromBookmark(id)).orElse(OptionT(fromShare(id))).value
def runQuery(bm: Option[ItemQuery], str: String): F[Unit] = def runQuery(bm: Option[ItemQuery], str: Option[String]): F[Unit] = {
ItemQueryParser.parse(str) match { val bmFtsQuery = bm.map(e => FulltextExtract.findFulltext(e.expr))
case Right(q) => val queryStrResult =
val expr = bm.map(b => AndExpr(Nel.of(b.expr, q.expr))).getOrElse(q.expr) str.map(search.parseQueryString(ctx.args.account, SearchMode.Normal, _))
val query = Query
.all(ctx.args.account)
.withFix(_.copy(query = Expr.ValidItemStates.some))
.withCond(_ => Query.QueryExpr(expr))
ctx.logger.debug(s"Running query: ${queryString(expr)}") *> cont(query) (bmFtsQuery, queryStrResult) match {
case (
Some(bmr: FulltextExtract.SuccessResult),
Some(QueryParseResult.Success(q, ftq))
) =>
val nq = bmr.getExprPart.map(q.andCond).getOrElse(q)
val nftq =
(bmr.getFulltextPart |+| Some(" ") |+| ftq).map(_.trim).filter(_.nonEmpty)
val r = QueryParseResult.Success(nq, nftq)
ctx.logger.debug(s"Running query: $r") *> cont(r)
case Left(err) => case (None, Some(r: QueryParseResult.Success)) =>
ctx.logger.error( ctx.logger.debug(s"Running query: $r") *> cont(r)
s"Item query is invalid, stopping: ${ctx.args.query.map(_.query)} - ${err.render}"
) case (Some(bmr: FulltextExtract.SuccessResult), None) =>
search.parseQueryString(ctx.args.account, SearchMode.Normal, "") match {
case QueryParseResult.Success(q, _) =>
val nq = bmr.getExprPart.map(q.andCond).getOrElse(q)
ctx.logger.debug(s"Running query: $nq") *>
cont(QueryParseResult.Success(nq, bmr.getFulltextPart))
case err =>
ctx.logger.error(s"Internal error: $err")
}
case (failure1, res2) =>
ctx.logger.error(s"One or more error reading queries: $failure1 and $res2")
} }
}
(ctx.args.bookmark, ctx.args.query) match { (ctx.args.bookmark, ctx.args.query) match {
case (Some(bm), Some(qstr)) => case (Some(bm), Some(qstr)) =>
ctx.logger.debug(s"Using bookmark $bm and query $qstr") *> ctx.logger.debug(s"Using bookmark $bm and query $qstr") *>
fromBookmarkOrShare(bm).flatMap(bq => runQuery(bq, qstr.query)) fromBookmarkOrShare(bm).flatMap(bq => runQuery(bq, qstr.query.some))
case (Some(bm), None) => case (Some(bm), None) =>
fromBookmarkOrShare(bm).flatMap { fromBookmarkOrShare(bm).flatMap {
case Some(bq) => case Some(bq) =>
val query = Query(Query.Fix(ctx.args.account, Some(bq.expr), None)) ctx.logger.debug(s"Using bookmark: ${queryString(bq.expr)}") *>
ctx.logger.debug(s"Using bookmark: ${queryString(bq.expr)}") *> cont(query) runQuery(bq.some, None)
case None => case None =>
ctx.logger.error( ctx.logger.error(
@ -119,7 +134,7 @@ object PeriodicQueryTask {
} }
case (None, Some(qstr)) => case (None, Some(qstr)) =>
ctx.logger.debug(s"Using query: ${qstr.query}") *> runQuery(None, qstr.query) ctx.logger.debug(s"Using query: ${qstr.query}") *> runQuery(None, qstr.query.some)
case (None, None) => case (None, None) =>
ctx.logger.error(s"No query provided for task $taskName!") ctx.logger.error(s"No query provided for task $taskName!")
@ -129,17 +144,14 @@ object PeriodicQueryTask {
def withItems[F[_]: Sync]( def withItems[F[_]: Sync](
ctx: Context[F, Args], ctx: Context[F, Args],
store: Store[F], store: Store[F],
search: OSearch[F],
limit: Int, limit: Int,
now: Timestamp now: Timestamp
)( )(
cont: Vector[ListItem] => F[Unit] cont: Vector[ListItem] => F[Unit]
): F[Unit] = ): F[Unit] =
withQuery(ctx, store) { query => withQuery(ctx, store, search) { qs =>
val items = store val items = search.search(0, now.toUtcDate.some, Batch.limit(limit))(qs.q, qs.ftq)
.transact(QItem.findItems(query, now.toUtcDate, 0, Batch.limit(limit)))
.compile
.to(Vector)
items.flatMap(cont) items.flatMap(cont)
} }

2
modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala
View File

@ -84,7 +84,7 @@ object BasicData {
) )
for { for {
items <- QItem items <- QItem
.findItems(q, now.toUtcDate, 25, Batch.limit(itemIds.size)) .queryItems(q, now.toUtcDate, 25, Batch.limit(itemIds.size), None)
.compile .compile
.to(Vector) .to(Vector)
} yield items.map(apply(now)) } yield items.map(apply(now))

3
modules/restserver/src/main/resources/reference.conf
View File

@ -70,9 +70,6 @@ docspell.server {
# In order to keep this low, a limit can be defined here. # In order to keep this low, a limit can be defined here.
max-note-length = 180 max-note-length = 180
feature-search-2 = true
# This defines whether the classification form in the collective # This defines whether the classification form in the collective
# settings is displayed or not. If all joex instances have document # settings is displayed or not. If all joex instances have document
# classification disabled, it makes sense to hide its settings from # classification disabled, it makes sense to hide its settings from

122
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@ -19,12 +19,16 @@ import docspell.backend.ops.OUpload.{UploadData, UploadMeta, UploadResult}
import docspell.backend.ops._ import docspell.backend.ops._
import docspell.common._ import docspell.common._
import docspell.common.syntax.all._ import docspell.common.syntax.all._
import docspell.ftsclient.FtsResult
import docspell.restapi.model._ import docspell.restapi.model._
import docspell.restserver.conv.Conversions._
import docspell.restserver.http4s.ContentDisposition import docspell.restserver.http4s.ContentDisposition
import docspell.store.qb.Batch import docspell.store.qb.Batch
import docspell.store.queries.{AttachmentLight => QAttachmentLight, IdRefCount} import docspell.store.queries.{
AttachmentLight => QAttachmentLight,
FieldStats => QFieldStats,
ItemFieldValue => QItemFieldValue,
TagCount => QTagCount,
_
}
import docspell.store.records._ import docspell.store.records._
import docspell.store.{AddResult, UpdateResult} import docspell.store.{AddResult, UpdateResult}
@ -34,7 +38,7 @@ import org.log4s.Logger
trait Conversions { trait Conversions {
def mkSearchStats(sum: OItemSearch.SearchSummary): SearchStats = def mkSearchStats(sum: SearchSummary): SearchStats =
SearchStats( SearchStats(
sum.count, sum.count,
mkTagCloud(sum.tags), mkTagCloud(sum.tags),
@ -53,7 +57,7 @@ trait Conversions {
def mkFolderStats(fs: docspell.store.queries.FolderCount): FolderStats = def mkFolderStats(fs: docspell.store.queries.FolderCount): FolderStats =
FolderStats(fs.id, fs.name, mkIdName(fs.owner), fs.count) FolderStats(fs.id, fs.name, mkIdName(fs.owner), fs.count)
def mkFieldStats(fs: docspell.store.queries.FieldStats): FieldStats = def mkFieldStats(fs: QFieldStats): FieldStats =
FieldStats( FieldStats(
fs.field.id, fs.field.id,
fs.field.name, fs.field.name,
@ -76,7 +80,7 @@ trait Conversions {
mkTagCloud(d.tags) mkTagCloud(d.tags)
) )
def mkTagCloud(tags: List[OCollective.TagCount]) = def mkTagCloud(tags: List[QTagCount]) =
TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count))) TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count)))
def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) = def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) =
@ -144,7 +148,7 @@ trait Conversions {
data.relatedItems.map(mkItemLight).toList data.relatedItems.map(mkItemLight).toList
) )
def mkItemFieldValue(v: OItemSearch.ItemFieldValue): ItemFieldValue = def mkItemFieldValue(v: QItemFieldValue): ItemFieldValue =
ItemFieldValue(v.fieldId, v.fieldName, v.fieldLabel, v.fieldType, v.value) ItemFieldValue(v.fieldId, v.fieldName, v.fieldLabel, v.fieldType, v.value)
def mkAttachment( def mkAttachment(
@ -173,28 +177,13 @@ trait Conversions {
OItemSearch.CustomValue(v.field, v.value) OItemSearch.CustomValue(v.field, v.value)
def mkItemList( def mkItemList(
v: Vector[OItemSearch.ListItem], v: Vector[ListItem],
batch: Batch, batch: Batch,
capped: Boolean capped: Boolean
): ItemLightList = { ): ItemLightList = {
val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7)) val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup = def mkGroup(g: (String, Vector[ListItem])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLight).toList)
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListFts(
v: Vector[OFulltext.FtsItem],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(item => item.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OFulltext.FtsItem])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLight).toList) ItemLightGroup(g._1, g._2.map(mkItemLight).toList)
val gs = val gs =
@ -203,13 +192,13 @@ trait Conversions {
} }
def mkItemListWithTags( def mkItemListWithTags(
v: Vector[OItemSearch.ListItemWithTags], v: Vector[ListItemWithTags],
batch: Batch, batch: Batch,
capped: Boolean capped: Boolean
): ItemLightList = { ): ItemLightList = {
val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7)) val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup = def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList) ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList)
val gs = val gs =
@ -217,50 +206,7 @@ trait Conversions {
ItemLightList(gs, batch.limit, batch.offset, capped) ItemLightList(gs, batch.limit, batch.offset, capped)
} }
def mkItemListWithTagsFts( def mkItemLight(i: ListItem): ItemLight =
v: Vector[OFulltext.FtsItemWithTags],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(ti => ti.item.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OFulltext.FtsItemWithTags])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList)
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListWithTagsFtsPlain(
v: Vector[OFulltext.FtsItemWithTags],
batch: Batch,
capped: Boolean
): ItemLightList =
if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
else
ItemLightList(
List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)),
batch.limit,
batch.offset,
capped
)
def mkItemListFtsPlain(
v: Vector[OFulltext.FtsItem],
batch: Batch,
capped: Boolean
): ItemLightList =
if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
else
ItemLightList(
List(ItemLightGroup("Results", v.map(mkItemLight).toList)),
batch.limit,
batch.offset,
capped
)
def mkItemLight(i: OItemSearch.ListItem): ItemLight =
ItemLight( ItemLight(
i.id, i.id,
i.name, i.name,
@ -282,13 +228,7 @@ trait Conversions {
Nil // highlight Nil // highlight
) )
def mkItemLight(i: OFulltext.FtsItem): ItemLight = { def mkItemLightWithTags(i: ListItemWithTags): ItemLight =
val il = mkItemLight(i.item)
val highlight = mkHighlight(i.ftsData)
il.copy(highlighting = highlight)
}
def mkItemLightWithTags(i: OItemSearch.ListItemWithTags): ItemLight =
mkItemLight(i.item) mkItemLight(i.item)
.copy( .copy(
tags = i.tags.map(mkTag), tags = i.tags.map(mkTag),
@ -300,22 +240,6 @@ trait Conversions {
def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight = def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount) AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount)
def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = {
val il = mkItemLightWithTags(i.item)
val highlight = mkHighlight(i.ftsData)
il.copy(highlighting = highlight)
}
private def mkHighlight(ftsData: OFulltext.FtsData): List[HighlightEntry] =
ftsData.items.filter(_.context.nonEmpty).sortBy(-_.score).map { fdi =>
fdi.matchData match {
case FtsResult.AttachmentData(_, aName) =>
HighlightEntry(aName, fdi.context)
case FtsResult.ItemData =>
HighlightEntry("Item", fdi.context)
}
}
// job // job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = { def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = {
@ -571,7 +495,7 @@ trait Conversions {
oid: Option[Ident], oid: Option[Ident],
pid: Option[Ident] pid: Option[Ident]
): F[RContact] = ): F[RContact] =
timeId.map { case (id, now) => Conversions.timeId.map { case (id, now) =>
RContact(id, c.value.trim, c.kind, pid, oid, now) RContact(id, c.value.trim, c.kind, pid, oid, now)
} }
@ -590,7 +514,7 @@ trait Conversions {
) )
def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] =
timeId.map { case (id, now) => Conversions.timeId.map { case (id, now) =>
RUser( RUser(
id, id,
u.login, u.login,
@ -625,7 +549,7 @@ trait Conversions {
Tag(rt.tagId, rt.name, rt.category, rt.created) Tag(rt.tagId, rt.name, rt.category, rt.created)
def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] =
timeId.map { case (id, now) => Conversions.timeId.map { case (id, now) =>
RTag(id, cid, t.name.trim, t.category.map(_.trim), now) RTag(id, cid, t.name.trim, t.category.map(_.trim), now)
} }
@ -653,7 +577,7 @@ trait Conversions {
) )
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map { case (id, now) => Conversions.timeId.map { case (id, now) =>
RSource( RSource(
id, id,
cid, cid,
@ -691,7 +615,7 @@ trait Conversions {
Equipment(re.eid, re.name, re.created, re.notes, re.use) Equipment(re.eid, re.name, re.created, re.notes, re.use)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map { case (id, now) => Conversions.timeId.map { case (id, now) =>
REquipment(id, cid, e.name.trim, now, now, e.notes, e.use) REquipment(id, cid, e.name.trim, now, now, e.notes, e.use)
} }
@ -785,7 +709,7 @@ trait Conversions {
header.mediaType.mainType, header.mediaType.mainType,
header.mediaType.subType, header.mediaType.subType,
None None
).withCharsetName(header.mediaType.extensions.get("charset").getOrElse("unknown")) ).withCharsetName(header.mediaType.extensions.getOrElse("charset", "unknown"))
} }
object Conversions extends Conversions { object Conversions extends Conversions {

148
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@ -13,12 +13,7 @@ import cats.implicits._
import docspell.backend.BackendApp import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken import docspell.backend.auth.AuthToken
import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue}
import docspell.backend.ops.OItemSearch.{Batch, Query}
import docspell.backend.ops.OSimpleSearch
import docspell.backend.ops.OSimpleSearch.StringSearchResult
import docspell.common._ import docspell.common._
import docspell.query.FulltextExtract.Result.TooMany
import docspell.query.FulltextExtract.Result.UnsupportedPosition
import docspell.restapi.model._ import docspell.restapi.model._
import docspell.restserver.Config import docspell.restserver.Config
import docspell.restserver.conv.Conversions import docspell.restserver.conv.Conversions
@ -27,11 +22,11 @@ import docspell.restserver.http4s.ClientRequestInfo
import docspell.restserver.http4s.Responses import docspell.restserver.http4s.Responses
import docspell.restserver.http4s.{QueryParam => QP} import docspell.restserver.http4s.{QueryParam => QP}
import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityDecoder._
import org.http4s.circe.CirceEntityEncoder._ import org.http4s.circe.CirceEntityEncoder._
import org.http4s.dsl.Http4sDsl import org.http4s.dsl.Http4sDsl
import org.http4s.headers._ import org.http4s.headers._
import org.http4s.{HttpRoutes, Response}
object ItemRoutes { object ItemRoutes {
def apply[F[_]: Async]( def apply[F[_]: Async](
@ -40,75 +35,12 @@ object ItemRoutes {
user: AuthToken user: AuthToken
): HttpRoutes[F] = { ): HttpRoutes[F] = {
val logger = docspell.logging.getLogger[F] val logger = docspell.logging.getLogger[F]
val searchPart = ItemSearchPart[F](backend, cfg, user) val searchPart = ItemSearchPart[F](backend.search, cfg, user)
val dsl = new Http4sDsl[F] {} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
searchPart.routes <+> searchPart.routes <+>
HttpRoutes.of { HttpRoutes.of {
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
offset
) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
.restrictLimitTo(cfg.maxItemPageSize)
val limitCapped = limit.exists(_ > cfg.maxItemPageSize)
val itemQuery = ItemQueryString(q)
val settings = OSimpleSearch.Settings(
batch,
cfg.fullTextSearch.enabled,
detailFlag.getOrElse(false),
cfg.maxNoteLength,
searchMode.getOrElse(SearchMode.Normal)
)
val fixQuery = Query.Fix(user.account, None, None)
searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
val itemQuery = ItemQueryString(q)
val fixQuery = Query.Fix(user.account, None, None)
val settings = OSimpleSearch.StatsSettings(
useFTS = cfg.fullTextSearch.enabled,
searchMode = searchMode.getOrElse(SearchMode.Normal)
)
searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
case req @ POST -> Root / "search" =>
for {
timed <- Duration.stopTime[F]
userQuery <- req.as[ItemQuery]
batch = Batch(
userQuery.offset.getOrElse(0),
userQuery.limit.getOrElse(cfg.maxItemPageSize)
).restrictLimitTo(
cfg.maxItemPageSize
)
limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
itemQuery = ItemQueryString(userQuery.query)
settings = OSimpleSearch.Settings(
batch,
cfg.fullTextSearch.enabled,
userQuery.withDetails.getOrElse(false),
cfg.maxNoteLength,
searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
)
fixQuery = Query.Fix(user.account, None, None)
resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
dur <- timed
_ <- logger.debug(s"Search request: ${dur.formatExact}")
} yield resp
case req @ POST -> Root / "searchStats" =>
for {
userQuery <- req.as[ItemQuery]
itemQuery = ItemQueryString(userQuery.query)
fixQuery = Query.Fix(user.account, None, None)
settings = OSimpleSearch.StatsSettings(
useFTS = cfg.fullTextSearch.enabled,
searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
)
resp <- searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
} yield resp
case GET -> Root / Ident(id) => case GET -> Root / Ident(id) =>
for { for {
item <- backend.itemSearch.findItem(id, user.account.collective) item <- backend.itemSearch.findItem(id, user.account.collective)
@ -412,82 +344,6 @@ object ItemRoutes {
} }
} }
def searchItems[F[_]: Sync](
backend: BackendApp[F],
dsl: Http4sDsl[F]
)(
settings: OSimpleSearch.Settings,
fixQuery: Query.Fix,
itemQuery: ItemQueryString,
limitCapped: Boolean
): F[Response[F]] = {
import dsl._
def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList =
if (res.indexOnly)
Conversions.mkItemListFtsPlain(res.items, settings.batch, limitCapped)
else Conversions.mkItemListFts(res.items, settings.batch, limitCapped)
def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList =
if (res.indexOnly)
Conversions.mkItemListWithTagsFtsPlain(res.items, settings.batch, limitCapped)
else Conversions.mkItemListWithTagsFts(res.items, settings.batch, limitCapped)
backend.simpleSearch
.searchByString(settings)(fixQuery, itemQuery)
.flatMap {
case StringSearchResult.Success(items) =>
Ok(
items.fold(
convertFts,
convertFtsFull,
els => Conversions.mkItemList(els, settings.batch, limitCapped),
els => Conversions.mkItemListWithTags(els, settings.batch, limitCapped)
)
)
case StringSearchResult.FulltextMismatch(TooMany) =>
BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
BadRequest(
BasicResult(
false,
"Fulltext search must be in root position or inside the first AND."
)
)
case StringSearchResult.ParseFailed(pf) =>
BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
}
}
def searchItemStats[F[_]: Sync](
backend: BackendApp[F],
dsl: Http4sDsl[F]
)(
settings: OSimpleSearch.StatsSettings,
fixQuery: Query.Fix,
itemQuery: ItemQueryString
): F[Response[F]] = {
import dsl._
backend.simpleSearch
.searchSummaryByString(settings)(fixQuery, itemQuery)
.flatMap {
case StringSearchResult.Success(summary) =>
Ok(Conversions.mkSearchStats(summary))
case StringSearchResult.FulltextMismatch(TooMany) =>
BadRequest(BasicResult(false, "Only one fulltext search term is allowed."))
case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
BadRequest(
BasicResult(
false,
"Fulltext search must be in root position or inside the first AND."
)
)
case StringSearchResult.ParseFailed(pf) =>
BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
}
}
implicit final class OptionString(opt: Option[String]) { implicit final class OptionString(opt: Option[String]) {
def notEmpty: Option[String] = def notEmpty: Option[String] =
opt.map(_.trim).filter(_.nonEmpty) opt.map(_.trim).filter(_.nonEmpty)

112
modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala
View File

@ -11,26 +11,30 @@ import java.time.LocalDate
import cats.effect._ import cats.effect._
import cats.syntax.all._ import cats.syntax.all._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken import docspell.backend.auth.AuthToken
import docspell.backend.ops.search.QueryParseResult import docspell.backend.ops.OShare
import docspell.common.{Duration, SearchMode, Timestamp} import docspell.backend.ops.OShare.ShareQuery
import docspell.backend.ops.search.{OSearch, QueryParseResult}
import docspell.common._
import docspell.query.FulltextExtract.Result import docspell.query.FulltextExtract.Result
import docspell.restapi.model._ import docspell.restapi.model._
import docspell.restserver.Config import docspell.restserver.Config
import docspell.restserver.conv.Conversions import docspell.restserver.conv.Conversions
import docspell.restserver.http4s.{QueryParam => QP} import docspell.restserver.http4s.{QueryParam => QP}
import docspell.store.qb.Batch import docspell.store.qb.Batch
import docspell.store.queries.ListItemWithTags import docspell.store.queries.{ListItemWithTags, SearchSummary}
import org.http4s.circe.CirceEntityCodec._ import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.Http4sDsl import org.http4s.dsl.Http4sDsl
import org.http4s.{HttpRoutes, Response} import org.http4s.{HttpRoutes, Response}
final class ItemSearchPart[F[_]: Async]( final class ItemSearchPart[F[_]: Async](
backend: BackendApp[F], searchOps: OSearch[F],
cfg: Config, cfg: Config,
authToken: AuthToken parseQuery: (SearchMode, String) => QueryParseResult,
changeSummary: SearchSummary => SearchSummary = identity,
searchPath: String = "search",
searchStatsPath: String = "searchStats"
) extends Http4sDsl[F] { ) extends Http4sDsl[F] {
private[this] val logger = docspell.logging.getLogger[F] private[this] val logger = docspell.logging.getLogger[F]
@ -39,9 +43,9 @@ final class ItemSearchPart[F[_]: Async](
if (!cfg.featureSearch2) HttpRoutes.empty if (!cfg.featureSearch2) HttpRoutes.empty
else else
HttpRoutes.of { HttpRoutes.of {
case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( case GET -> Root / `searchPath` :? QP.Query(q) :? QP.Limit(limit) :?
offset QP.Offset(offset) :? QP.WithDetails(detailFlag) :?
) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => QP.SearchKind(searchMode) =>
val userQuery = val userQuery =
ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse("")) ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse(""))
for { for {
@ -49,7 +53,7 @@ final class ItemSearchPart[F[_]: Async](
resp <- search(userQuery, today) resp <- search(userQuery, today)
} yield resp } yield resp
case req @ POST -> Root / "search" => case req @ POST -> Root / `searchPath` =>
for { for {
timed <- Duration.stopTime[F] timed <- Duration.stopTime[F]
userQuery <- req.as[ItemQuery] userQuery <- req.as[ItemQuery]
@ -59,14 +63,15 @@ final class ItemSearchPart[F[_]: Async](
_ <- logger.debug(s"Search request: ${dur.formatExact}") _ <- logger.debug(s"Search request: ${dur.formatExact}")
} yield resp } yield resp
case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => case GET -> Root / `searchStatsPath` :? QP.Query(q) :?
QP.SearchKind(searchMode) =>
val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse("")) val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse(""))
for { for {
today <- Timestamp.current[F].map(_.toUtcDate) today <- Timestamp.current[F].map(_.toUtcDate)
resp <- searchStats(userQuery, today) resp <- searchStats(userQuery, today)
} yield resp } yield resp
case req @ POST -> Root / "searchStats" => case req @ POST -> Root / `searchStatsPath` =>
for { for {
timed <- Duration.stopTime[F] timed <- Duration.stopTime[F]
userQuery <- req.as[ItemQuery] userQuery <- req.as[ItemQuery]
@ -84,8 +89,8 @@ final class ItemSearchPart[F[_]: Async](
identity, identity,
res => res =>
for { for {
summary <- backend.search.searchSummary(today)(res.q, res.ftq) summary <- searchOps.searchSummary(today.some)(res.q, res.ftq)
resp <- Ok(Conversions.mkSearchStats(summary)) resp <- Ok(Conversions.mkSearchStats(changeSummary(summary)))
} yield resp } yield resp
) )
} }
@ -103,8 +108,9 @@ final class ItemSearchPart[F[_]: Async](
identity, identity,
res => res =>
for { for {
items <- backend.search _ <- logger.warn(s"Searching with query: $res")
.searchSelect(details, cfg.maxNoteLength, today, batch)( items <- searchOps
.searchSelect(details, cfg.maxNoteLength, today.some, batch)(
res.q, res.q,
res.ftq res.ftq
) )
@ -122,29 +128,10 @@ final class ItemSearchPart[F[_]: Async](
userQuery: ItemQuery, userQuery: ItemQuery,
mode: SearchMode mode: SearchMode
): Either[F[Response[F]], QueryParseResult.Success] = ): Either[F[Response[F]], QueryParseResult.Success] =
backend.search.parseQueryString(authToken.account, mode, userQuery.query) match { convertParseResult(
case s: QueryParseResult.Success => parseQuery(mode, userQuery.query)
Right(s.withFtsEnabled(cfg.fullTextSearch.enabled)) .map(_.withFtsEnabled(cfg.fullTextSearch.enabled))
)
case QueryParseResult.ParseFailed(err) =>
Left(BadRequest(BasicResult(false, s"Invalid query: $err")))
case QueryParseResult.FulltextMismatch(Result.TooMany) =>
Left(
BadRequest(
BasicResult(false, "Only one fulltext search expression is allowed.")
)
)
case QueryParseResult.FulltextMismatch(Result.UnsupportedPosition) =>
Left(
BadRequest(
BasicResult(
false,
"A fulltext search may only appear in the root and expression."
)
)
)
}
def convert( def convert(
items: Vector[ListItemWithTags], items: Vector[ListItemWithTags],
@ -202,13 +189,56 @@ final class ItemSearchPart[F[_]: Async](
Nil Nil
} }
) )
def convertParseResult(
r: QueryParseResult
): Either[F[Response[F]], QueryParseResult.Success] =
r match {
case s: QueryParseResult.Success =>
Right(s)
case QueryParseResult.ParseFailed(err) =>
BadRequest(BasicResult(false, s"Invalid query: $err")).asLeft
case QueryParseResult.FulltextMismatch(Result.TooMany) =>
BadRequest(
BasicResult(false, "Only one fulltext search expression is allowed.")
).asLeft
case QueryParseResult.FulltextMismatch(Result.UnsupportedPosition) =>
BadRequest(
BasicResult(
false,
"A fulltext search may only appear in the root and expression."
)
).asLeft
}
} }
object ItemSearchPart { object ItemSearchPart {
def apply[F[_]: Async]( def apply[F[_]: Async](
backend: BackendApp[F], search: OSearch[F],
cfg: Config, cfg: Config,
token: AuthToken token: AuthToken
): ItemSearchPart[F] = ): ItemSearchPart[F] =
new ItemSearchPart[F](backend, cfg, token) new ItemSearchPart[F](
search,
cfg,
(m, s) => search.parseQueryString(token.account, m, s)
)
def apply[F[_]: Async](
search: OSearch[F],
share: OShare[F],
cfg: Config,
shareQuery: ShareQuery
): ItemSearchPart[F] =
new ItemSearchPart[F](
search,
cfg,
(_, s) => share.parseQuery(shareQuery, s),
changeSummary = _.onlyExisting,
searchPath = "query",
searchStatsPath = "stats"
)
} }

98
modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala
View File

@ -6,25 +6,14 @@
package docspell.restserver.routes package docspell.restserver.routes
import cats.data.Kleisli
import cats.effect._ import cats.effect._
import cats.implicits._
import docspell.backend.BackendApp import docspell.backend.BackendApp
import docspell.backend.auth.ShareToken import docspell.backend.auth.ShareToken
import docspell.backend.ops.OSimpleSearch
import docspell.backend.ops.OSimpleSearch.StringSearchResult
import docspell.common._
import docspell.query.FulltextExtract.Result.{TooMany, UnsupportedPosition}
import docspell.restapi.model._
import docspell.restserver.Config import docspell.restserver.Config
import docspell.restserver.conv.Conversions
import docspell.store.qb.Batch
import docspell.store.queries.{Query, SearchSummary}
import org.http4s.circe.CirceEntityDecoder._ import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityEncoder._
import org.http4s.dsl.Http4sDsl
import org.http4s.{HttpRoutes, Response}
object ShareSearchRoutes { object ShareSearchRoutes {
@ -32,80 +21,13 @@ object ShareSearchRoutes {
backend: BackendApp[F], backend: BackendApp[F],
cfg: Config, cfg: Config,
token: ShareToken token: ShareToken
): HttpRoutes[F] = { ): HttpRoutes[F] =
val logger = docspell.logging.getLogger[F] Kleisli { req =>
for {
val dsl = new Http4sDsl[F] {} shareQuery <- backend.share.findShareQuery(token.id)
import dsl._ searchPart = ItemSearchPart(backend.search, backend.share, cfg, shareQuery)
routes = searchPart.routes
HttpRoutes.of { resp <- routes(req)
case req @ POST -> Root / "query" => } yield resp
backend.share
.findShareQuery(token.id)
.semiflatMap { share =>
for {
userQuery <- req.as[ItemQuery]
batch = Batch(
userQuery.offset.getOrElse(0),
userQuery.limit.getOrElse(cfg.maxItemPageSize)
).restrictLimitTo(
cfg.maxItemPageSize
)
limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
itemQuery = ItemQueryString(userQuery.query)
settings = OSimpleSearch.Settings(
batch,
cfg.fullTextSearch.enabled,
userQuery.withDetails.getOrElse(false),
cfg.maxNoteLength,
searchMode = SearchMode.Normal
)
account = share.account
fixQuery = Query.Fix(account, Some(share.query.expr), None)
_ <- logger.debug(s"Searching in share ${share.id.id}: ${userQuery.query}")
resp <- ItemRoutes.searchItems(backend, dsl)(
settings,
fixQuery,
itemQuery,
limitCapped
)
} yield resp
}
.getOrElseF(NotFound())
case req @ POST -> Root / "stats" =>
for {
userQuery <- req.as[ItemQuery]
itemQuery = ItemQueryString(userQuery.query)
settings = OSimpleSearch.StatsSettings(
useFTS = cfg.fullTextSearch.enabled,
searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
)
stats <- backend.share.searchSummary(settings)(token.id, itemQuery).value
resp <- stats.map(mkSummaryResponse(dsl)).getOrElse(NotFound())
} yield resp
} }
}
def mkSummaryResponse[F[_]: Sync](
dsl: Http4sDsl[F]
)(r: StringSearchResult[SearchSummary]): F[Response[F]] = {
import dsl._
r match {
case StringSearchResult.Success(summary) =>
Ok(Conversions.mkSearchStats(summary))
case StringSearchResult.FulltextMismatch(TooMany) =>
BadRequest(BasicResult(false, "Fulltext search is not possible in this share."))
case StringSearchResult.FulltextMismatch(UnsupportedPosition) =>
BadRequest(
BasicResult(
false,
"Fulltext search must be in root position or inside the first AND."
)
)
case StringSearchResult.ParseFailed(pf) =>
BadRequest(BasicResult(false, s"Error reading query: ${pf.render}"))
}
}
} }

57
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -73,14 +73,6 @@ object QItem extends FtsSupport {
sql.query[ListItem].stream sql.query[ListItem].stream
} }
// ----
def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] =
Select(count(a.id).s, from(a), a.itemId.in(items)).build
.query[Int]
.unique
.map(_ + items.size)
def findItem(id: Ident, collective: Ident): ConnectionIO[Option[ItemData]] = { def findItem(id: Ident, collective: Ident): ConnectionIO[Option[ItemData]] = {
val ref = RItem.as("ref") val ref = RItem.as("ref")
val cq = val cq =
@ -314,14 +306,6 @@ object QItem extends FtsSupport {
Condition.unit Condition.unit
} }
def findItems(
q: Query,
today: LocalDate,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] =
queryItems(q, today, maxNoteLen, batch, None)
def searchStats(today: LocalDate, ftsTable: Option[RFtsResult.Table])( def searchStats(today: LocalDate, ftsTable: Option[RFtsResult.Table])(
q: Query q: Query
): ConnectionIO[SearchSummary] = ): ConnectionIO[SearchSummary] =
@ -524,47 +508,6 @@ object QItem extends FtsSupport {
} }
} }
def findSelectedItems(
q: Query,
today: LocalDate,
maxNoteLen: Int,
items: Set[SelectedItem]
): Stream[ConnectionIO, ListItem] =
if (items.isEmpty) Stream.empty
else {
val i = RItem.as("i")
object Tids extends TableDef {
val tableName = "tids"
val alias = Some("tw")
val itemId = Column[Ident]("item_id", this)
val weight = Column[Double]("weight", this)
val all = Vector[Column[_]](itemId, weight)
}
val cte =
CteBind(
Tids,
Tids.all,
Select.RawSelect(
fr"VALUES" ++
items
.map(it => fr"(${it.itemId}, ${it.weight})")
.reduce((r, e) => r ++ fr"," ++ e)
)
)
val from = findItemsBase(q.fix, today, maxNoteLen, None)
.appendCte(cte)
.appendSelect(Tids.weight.s)
.changeFrom(_.innerJoin(Tids, Tids.itemId === i.id))
.orderBy(Tids.weight.desc)
.build
logger.stream.trace(s"fts query: $from").drain ++
from.query[ListItem].stream
}
/** Same as `findItems` but resolves the tags for each item. Note that this is /** Same as `findItems` but resolves the tags for each item. Note that this is
* implemented by running an additional query per item. * implemented by running an additional query per item.
*/ */

9
modules/store/src/main/scala/docspell/store/queries/Query.scala
View File

@ -17,6 +17,15 @@ case class Query(fix: Query.Fix, cond: Query.QueryCond) {
def withCond(f: Query.QueryCond => Query.QueryCond): Query = def withCond(f: Query.QueryCond => Query.QueryCond): Query =
copy(cond = f(cond)) copy(cond = f(cond))
def andCond(c: ItemQuery.Expr): Query =
withCond {
case Query.QueryExpr(Some(q)) =>
Query.QueryExpr(ItemQuery.Expr.and(q, c))
case Query.QueryExpr(None) =>
Query.QueryExpr(c)
}
def withOrder(orderAsc: RItem.Table => Column[_]): Query = def withOrder(orderAsc: RItem.Table => Column[_]): Query =
withFix(_.copy(order = Some(_.byItemColumnAsc(orderAsc)))) withFix(_.copy(order = Some(_.byItemColumnAsc(orderAsc))))