TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-04 06:05:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Return item notes with search results

Browse Source 
In order to not make the response very large, a admin can define a
limit on how much to return.
This commit is contained in:
Eike Kettner 2020-08-04 22:45:35 +02:00
parent f1e776ae3d
commit 09d74b7e80
11 changed files with 75 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@ -15,20 +15,20 @@ import docspell.store.records.RJob
trait OFulltext[F[_]] {
def findItems(
def findItems(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
fts: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@ -92,7 +92,7 @@ object OFulltext {
else queue.insertIfNew(job) *> joex.notifyAllNodes
} yield ()
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
ftsQ: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@ -120,7 +120,7 @@ object OFulltext {
.transact(
QItem.findItemsWithTags(
account.collective,
QItem.findSelectedItems(QItem.Query.empty(account), select)
QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select)
)
)
.take(batch.limit.toLong)
@ -133,15 +133,23 @@ object OFulltext {
} yield res
}
def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem])
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItems(maxNoteLen),
convertFtsData[ListItem]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map({ case (li, fd) => FtsItem(li, fd) })
.compile
.toVector
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
ftsQ: FtsInput,
batch: Batch
@ -150,7 +158,7 @@ object OFulltext {
q,
ftsQ,
batch.first,
itemSearch.findItemsWithTags,
itemSearch.findItemsWithTags(maxNoteLen),
convertFtsData[ListItemWithTags]
)
.drop(batch.offset.toLong)

16
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@ -17,10 +17,12 @@ import doobie.implicits._
trait OItemSearch[F[_]] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
@ -97,14 +99,16 @@ object OItemSearch {
.transact(QItem.findItem(id))
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, batch).take(batch.limit.toLong))
.transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, batch)
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, maxNoteLen: Int, batch)
store
.transact(
QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)

2
modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
View File

@ -82,7 +82,7 @@ object NotifyDueItemsTask {
)
res <-
ctx.store
.transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong))
.transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
.compile
.toVector
} yield res

4
modules/restapi/src/main/resources/docspell-openapi.yml
View File

@ -3860,6 +3860,10 @@ components:
type: array
items:
$ref: "#/components/schemas/Tag"
notes:
description: |
Some prefix of the item notes.
type: string
highlighting:
description: |
Optional contextual information of a search query. Each

6
modules/restserver/src/main/resources/reference.conf
View File

@ -24,6 +24,12 @@ docspell.server {
# depending on the available resources.
max-item-page-size = 200
# The number of characters to return for each item notes when
# searching. Item notes may be very long, when returning them with
# all the results from a search, they add quite some data to return.
# In order to keep this low, a limit can be defined here.
max-note-length = 180
# Authentication.
auth {

1
modules/restserver/src/main/scala/docspell/restserver/Config.scala
View File

@ -16,6 +16,7 @@ case class Config(
auth: Login.Config,
integrationEndpoint: Config.IntegrationEndpoint,
maxItemPageSize: Int,
maxNoteLength: Int,
fullTextSearch: Config.FullTextSearch
)

1
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@ -197,6 +197,7 @@ trait Conversions {
i.folder.map(mkIdName),
i.fileCount,
Nil,
i.notes,
Nil
)

10
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@ -40,7 +40,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItems(
items <- backend.fulltext.findItems(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@ -49,7 +49,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItems(
items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -67,7 +67,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItemsWithTags(
items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@ -76,7 +76,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItemsWithTags(
items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -92,7 +92,7 @@ object ItemRoutes {
case q if q.length > 1 =>
val ftsIn = OFulltext.FtsInput(q)
for {
items <- backend.fulltext.findIndexOnly(
items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
ftsIn,
user.account,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)

4
modules/store/src/main/scala/docspell/store/impl/Column.scala
View File

@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def decrement[A: Put](a: A): Fragment =
f ++ fr"=" ++ f ++ fr"- $a"
def substring(from: Int, many: Int): Fragment =
if (many <= 0 || from < 0) fr"${""}"
else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)"
}

18
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -156,7 +156,8 @@ object QItem {
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef],
folder: Option[IdRef]
folder: Option[IdRef],
notes: Option[String]
)
case class Query(
@ -228,6 +229,7 @@ object QItem {
private def findItemsBase(
q: Query,
distinct: Boolean,
noteMaxLen: Int,
moreCols: Seq[Fragment],
ctes: (String, Fragment)*
): Fragment = {
@ -264,6 +266,9 @@ object QItem {
EC.name.prefix("e1").f,
FC.id.prefix("f1").f,
FC.name.prefix("f1").f,
// sql uses 1 for first character
IC.notes.prefix("i").substring(1, noteMaxLen),
// last column is only for sorting
q.orderAsc match {
case Some(co) =>
coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
@ -307,14 +312,16 @@ object QItem {
fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1"))
}
def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = {
def findItems(
q: Query,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns
val PC = RPerson.Columns
val OC = ROrganization.Columns
val EC = REquipment.Columns
val query = findItemsBase(q, true, Seq.empty)
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude
.map(tid =>
@ -404,6 +411,7 @@ object QItem {
if (batch == Batch.all) Fragment.empty
else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
val frag =
query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
logger.trace(s"List $batch items: $frag")
@ -413,6 +421,7 @@ object QItem {
case class SelectedItem(itemId: Ident, weight: Double)
def findSelectedItems(
q: Query,
maxNoteLen: Int,
items: Set[SelectedItem]
): Stream[ConnectionIO, ListItem] =
if (items.isEmpty) Stream.empty
@ -425,6 +434,7 @@ object QItem {
val from = findItemsBase(
q,
true,
maxNoteLen,
Seq(fr"tids.weight"),
("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
) ++

12
nix/module-server.nix
View File

@ -14,6 +14,7 @@ let
app-id = "rest1";
base-url = "http://localhost:7880";
max-item-page-size = 200;
max-note-length = 180;
bind = {
address = "localhost";
port = 7880;
@ -124,6 +125,17 @@ in {
'';
};
max-note-length = mkOption {
type = types.int;
default = defaults.max-note-length;
description = ''
The number of characters to return for each item notes when
searching. Item notes may be very long, when returning them with
all the results from a search, they add quite some data to return.
In order to keep this low, a limit can be defined here.
'';
};
bind = mkOption {
type = types.submodule({
options = {