TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-02-15 20:33:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Include limit-capped flag with search response

Browse Source 
The server defines a `limit` value and search requests are capped to
this limit if their requested value exceeds it. If this happens it is
now returned with the search response (clients can print a warning).

Closes: #1358
This commit is contained in:
eikek 2022-05-26 22:24:56 +02:00
parent f8baf44c09
commit 50edf13f94
5 changed files with 106 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
modules/restapi/src/main/resources/docspell-openapi.yml
View File

@ -8591,11 +8591,33 @@ components:
A list of item details.
required:
- groups
- limit
- offset
- limitCapped
properties:
groups:
type: array
items:
$ref: "#/components/schemas/ItemLightGroup"
limit:
type: integer
format: int32
description: |
Returns the `limit` value as used for this search. This
can deviate from the requested limit, if it exceeds the
server defined maximum. See `limitCapped`.
offset:
type: integer
format: int32
description: |
The `offset` value used for this search.
limitCapped:
type: boolean
description: |
The server defines a maximum `limit` value. If the
requested `limit` exceeds the server defined one, this
flag is set to true. The limit used for the query is
returned with this response.
ItemLightGroup:
description: |
A group of items.

65
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@ -23,6 +23,7 @@ import docspell.ftsclient.FtsResult
import docspell.restapi.model._
import docspell.restserver.conv.Conversions._
import docspell.restserver.http4s.ContentDisposition
import docspell.store.qb.Batch
import docspell.store.queries.{AttachmentLight => QAttachmentLight, IdRefCount}
import docspell.store.records._
import docspell.store.{AddResult, UpdateResult}
@ -171,7 +172,11 @@ trait Conversions {
def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue =
OItemSearch.CustomValue(v.field, v.value)
def mkItemList(v: Vector[OItemSearch.ListItem]): ItemLightList = {
def mkItemList(
v: Vector[OItemSearch.ListItem],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup =
@ -179,10 +184,14 @@ trait Conversions {
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListFts(v: Vector[OFulltext.FtsItem]): ItemLightList = {
def mkItemListFts(
v: Vector[OFulltext.FtsItem],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(item => item.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OFulltext.FtsItem])): ItemLightGroup =
@ -190,10 +199,14 @@ trait Conversions {
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListWithTags(v: Vector[OItemSearch.ListItemWithTags]): ItemLightList = {
def mkItemListWithTags(
v: Vector[OItemSearch.ListItemWithTags],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup =
@ -201,10 +214,14 @@ trait Conversions {
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListWithTagsFts(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList = {
def mkItemListWithTagsFts(
v: Vector[OFulltext.FtsItemWithTags],
batch: Batch,
capped: Boolean
): ItemLightList = {
val groups = v.groupBy(ti => ti.item.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OFulltext.FtsItemWithTags])): ItemLightGroup =
@ -212,16 +229,36 @@ trait Conversions {
val gs =
groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
ItemLightList(gs, batch.limit, batch.offset, capped)
}
def mkItemListWithTagsFtsPlain(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList =
if (v.isEmpty) ItemLightList(Nil)
else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)))
def mkItemListWithTagsFtsPlain(
v: Vector[OFulltext.FtsItemWithTags],
batch: Batch,
capped: Boolean
): ItemLightList =
if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
else
ItemLightList(
List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)),
batch.limit,
batch.offset,
capped
)
def mkItemListFtsPlain(v: Vector[OFulltext.FtsItem]): ItemLightList =
if (v.isEmpty) ItemLightList(Nil)
else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLight).toList)))
def mkItemListFtsPlain(
v: Vector[OFulltext.FtsItem],
batch: Batch,
capped: Boolean
): ItemLightList =
if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
else
ItemLightList(
List(ItemLightGroup("Results", v.map(mkItemLight).toList)),
batch.limit,
batch.offset,
capped
)
def mkItemLight(i: OItemSearch.ListItem): ItemLight =
ItemLight(

40
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@ -50,6 +50,7 @@ object ItemRoutes {
) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
.restrictLimitTo(cfg.maxItemPageSize)
val limitCapped = limit.exists(_ > cfg.maxItemPageSize)
val itemQuery = ItemQueryString(q)
val settings = OSimpleSearch.Settings(
batch,
@ -59,7 +60,7 @@ object ItemRoutes {
searchMode.getOrElse(SearchMode.Normal)
)
val fixQuery = Query.Fix(user.account, None, None)
searchItems(backend, dsl)(settings, fixQuery, itemQuery)
searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
val itemQuery = ItemQueryString(q)
@ -79,6 +80,7 @@ object ItemRoutes {
).restrictLimitTo(
cfg.maxItemPageSize
)
limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
itemQuery = ItemQueryString(userQuery.query)
settings = OSimpleSearch.Settings(
batch,
@ -88,7 +90,7 @@ object ItemRoutes {
searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
)
fixQuery = Query.Fix(user.account, None, None)
resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery)
resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
} yield resp
case req @ POST -> Root / "searchStats" =>
@ -106,19 +108,20 @@ object ItemRoutes {
case req @ POST -> Root / "searchIndex" =>
for {
mask <- req.as[ItemQuery]
limitCapped = mask.limit.exists(_ > cfg.maxItemPageSize)
resp <- mask.query match {
case q if q.length > 1 =>
val ftsIn = OFulltext.FtsInput(q)
val batch = Batch(
mask.offset.getOrElse(0),
mask.limit.getOrElse(cfg.maxItemPageSize)
).restrictLimitTo(cfg.maxItemPageSize)
for {
items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
ftsIn,
user.account,
Batch(
mask.offset.getOrElse(0),
mask.limit.getOrElse(cfg.maxItemPageSize)
).restrictLimitTo(cfg.maxItemPageSize)
items <- backend.fulltext
.findIndexOnly(cfg.maxNoteLength)(ftsIn, user.account, batch)
ok <- Ok(
Conversions.mkItemListWithTagsFtsPlain(items, batch, limitCapped)
)
ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items))
} yield ok
case _ =>
@ -429,17 +432,20 @@ object ItemRoutes {
)(
settings: OSimpleSearch.Settings,
fixQuery: Query.Fix,
itemQuery: ItemQueryString
itemQuery: ItemQueryString,
limitCapped: Boolean
): F[Response[F]] = {
import dsl._
def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList =
if (res.indexOnly) Conversions.mkItemListFtsPlain(res.items)
else Conversions.mkItemListFts(res.items)
if (res.indexOnly)
Conversions.mkItemListFtsPlain(res.items, settings.batch, limitCapped)
else Conversions.mkItemListFts(res.items, settings.batch, limitCapped)
def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList =
if (res.indexOnly) Conversions.mkItemListWithTagsFtsPlain(res.items)
else Conversions.mkItemListWithTagsFts(res.items)
if (res.indexOnly)
Conversions.mkItemListWithTagsFtsPlain(res.items, settings.batch, limitCapped)
else Conversions.mkItemListWithTagsFts(res.items, settings.batch, limitCapped)
backend.simpleSearch
.searchByString(settings)(fixQuery, itemQuery)
@ -449,8 +455,8 @@ object ItemRoutes {
items.fold(
convertFts,
convertFtsFull,
Conversions.mkItemList,
Conversions.mkItemListWithTags
els => Conversions.mkItemList(els, settings.batch, limitCapped),
els => Conversions.mkItemListWithTags(els, settings.batch, limitCapped)
)
)
case StringSearchResult.FulltextMismatch(TooMany) =>

8
modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala
View File

@ -51,6 +51,7 @@ object ShareSearchRoutes {
).restrictLimitTo(
cfg.maxItemPageSize
)
limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
itemQuery = ItemQueryString(userQuery.query)
settings = OSimpleSearch.Settings(
batch,
@ -62,7 +63,12 @@ object ShareSearchRoutes {
account = share.account
fixQuery = Query.Fix(account, Some(share.query.expr), None)
_ <- logger.debug(s"Searching in share ${share.id.id}: ${userQuery.query}")
resp <- ItemRoutes.searchItems(backend, dsl)(settings, fixQuery, itemQuery)
resp <- ItemRoutes.searchItems(backend, dsl)(
settings,
fixQuery,
itemQuery,
limitCapped
)
} yield resp
}
.getOrElseF(NotFound())

6
modules/webapp/src/main/elm/Data/Items.elm
View File

@ -64,10 +64,10 @@ concat l0 l1 =
suff =
List.drop 1 l1.groups
in
ItemLightList (prev ++ (ng :: suff))
ItemLightList (prev ++ (ng :: suff)) 0 0 False
else
ItemLightList (l0.groups ++ l1.groups)
ItemLightList (l0.groups ++ l1.groups) 0 0 False
first : ItemLightList -> Maybe ItemLight
@ -121,7 +121,7 @@ replaceIn origin replacements =
|> ItemLightGroup g.name
in
List.map replaceGroup origin.groups
|> ItemLightList
|> (\els -> ItemLightList els origin.limit origin.offset origin.limitCapped)