From 50edf13f941acbbf3b1ce5b86b8e23fb6a3ca4e3 Mon Sep 17 00:00:00 2001 From: eikek Date: Thu, 26 May 2022 22:24:56 +0200 Subject: [PATCH] Include limit-capped flag with search response The server defines a `limit` value and search requests are capped to this limit if their requested value exceeds it. If this happens it is now returned with the search response (clients can print a warning). Closes: #1358 --- .../src/main/resources/docspell-openapi.yml | 22 +++++++ .../restserver/conv/Conversions.scala | 65 +++++++++++++++---- .../restserver/routes/ItemRoutes.scala | 40 +++++++----- .../restserver/routes/ShareSearchRoutes.scala | 8 ++- modules/webapp/src/main/elm/Data/Items.elm | 6 +- 5 files changed, 106 insertions(+), 35 deletions(-) diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 9bef908f..401534b1 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -8591,11 +8591,33 @@ components: A list of item details. required: - groups + - limit + - offset + - limitCapped properties: groups: type: array items: $ref: "#/components/schemas/ItemLightGroup" + limit: + type: integer + format: int32 + description: | + Returns the `limit` value as used for this search. This + can deviate from the requested limit, if it exceeds the + server defined maximum. See `limitCapped`. + offset: + type: integer + format: int32 + description: | + The `offset` value used for this search. + limitCapped: + type: boolean + description: | + The server defines a maximum `limit` value. If the + requested `limit` exceeds the server defined one, this + flag is set to true. The limit used for the query is + returned with this response. ItemLightGroup: description: | A group of items. diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 26c95dd2..28421776 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -23,6 +23,7 @@ import docspell.ftsclient.FtsResult import docspell.restapi.model._ import docspell.restserver.conv.Conversions._ import docspell.restserver.http4s.ContentDisposition +import docspell.store.qb.Batch import docspell.store.queries.{AttachmentLight => QAttachmentLight, IdRefCount} import docspell.store.records._ import docspell.store.{AddResult, UpdateResult} @@ -171,7 +172,11 @@ trait Conversions { def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue = OItemSearch.CustomValue(v.field, v.value) - def mkItemList(v: Vector[OItemSearch.ListItem]): ItemLightList = { + def mkItemList( + v: Vector[OItemSearch.ListItem], + batch: Batch, + capped: Boolean + ): ItemLightList = { val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7)) def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup = @@ -179,10 +184,14 @@ trait Conversions { val gs = groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs) + ItemLightList(gs, batch.limit, batch.offset, capped) } - def mkItemListFts(v: Vector[OFulltext.FtsItem]): ItemLightList = { + def mkItemListFts( + v: Vector[OFulltext.FtsItem], + batch: Batch, + capped: Boolean + ): ItemLightList = { val groups = v.groupBy(item => item.item.date.toUtcDate.toString.substring(0, 7)) def mkGroup(g: (String, Vector[OFulltext.FtsItem])): ItemLightGroup = @@ -190,10 +199,14 @@ trait Conversions { val gs = groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs) + ItemLightList(gs, batch.limit, batch.offset, capped) } - def mkItemListWithTags(v: Vector[OItemSearch.ListItemWithTags]): ItemLightList = { + def mkItemListWithTags( + v: Vector[OItemSearch.ListItemWithTags], + batch: Batch, + capped: Boolean + ): ItemLightList = { val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7)) def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup = @@ -201,10 +214,14 @@ trait Conversions { val gs = groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs) + ItemLightList(gs, batch.limit, batch.offset, capped) } - def mkItemListWithTagsFts(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList = { + def mkItemListWithTagsFts( + v: Vector[OFulltext.FtsItemWithTags], + batch: Batch, + capped: Boolean + ): ItemLightList = { val groups = v.groupBy(ti => ti.item.item.date.toUtcDate.toString.substring(0, 7)) def mkGroup(g: (String, Vector[OFulltext.FtsItemWithTags])): ItemLightGroup = @@ -212,16 +229,36 @@ trait Conversions { val gs = groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) - ItemLightList(gs) + ItemLightList(gs, batch.limit, batch.offset, capped) } - def mkItemListWithTagsFtsPlain(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList = - if (v.isEmpty) ItemLightList(Nil) - else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList))) + def mkItemListWithTagsFtsPlain( + v: Vector[OFulltext.FtsItemWithTags], + batch: Batch, + capped: Boolean + ): ItemLightList = + if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped) + else + ItemLightList( + List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)), + batch.limit, + batch.offset, + capped + ) - def mkItemListFtsPlain(v: Vector[OFulltext.FtsItem]): ItemLightList = - if (v.isEmpty) ItemLightList(Nil) - else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLight).toList))) + def mkItemListFtsPlain( + v: Vector[OFulltext.FtsItem], + batch: Batch, + capped: Boolean + ): ItemLightList = + if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped) + else + ItemLightList( + List(ItemLightGroup("Results", v.map(mkItemLight).toList)), + batch.limit, + batch.offset, + capped + ) def mkItemLight(i: OItemSearch.ListItem): ItemLight = ItemLight( diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index bcabb4c7..2c589f9c 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -50,6 +50,7 @@ object ItemRoutes { ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize)) .restrictLimitTo(cfg.maxItemPageSize) + val limitCapped = limit.exists(_ > cfg.maxItemPageSize) val itemQuery = ItemQueryString(q) val settings = OSimpleSearch.Settings( batch, @@ -59,7 +60,7 @@ object ItemRoutes { searchMode.getOrElse(SearchMode.Normal) ) val fixQuery = Query.Fix(user.account, None, None) - searchItems(backend, dsl)(settings, fixQuery, itemQuery) + searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped) case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => val itemQuery = ItemQueryString(q) @@ -79,6 +80,7 @@ object ItemRoutes { ).restrictLimitTo( cfg.maxItemPageSize ) + limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize) itemQuery = ItemQueryString(userQuery.query) settings = OSimpleSearch.Settings( batch, @@ -88,7 +90,7 @@ object ItemRoutes { searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal) ) fixQuery = Query.Fix(user.account, None, None) - resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery) + resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped) } yield resp case req @ POST -> Root / "searchStats" => @@ -106,19 +108,20 @@ object ItemRoutes { case req @ POST -> Root / "searchIndex" => for { mask <- req.as[ItemQuery] + limitCapped = mask.limit.exists(_ > cfg.maxItemPageSize) resp <- mask.query match { case q if q.length > 1 => val ftsIn = OFulltext.FtsInput(q) + val batch = Batch( + mask.offset.getOrElse(0), + mask.limit.getOrElse(cfg.maxItemPageSize) + ).restrictLimitTo(cfg.maxItemPageSize) for { - items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( - ftsIn, - user.account, - Batch( - mask.offset.getOrElse(0), - mask.limit.getOrElse(cfg.maxItemPageSize) - ).restrictLimitTo(cfg.maxItemPageSize) + items <- backend.fulltext + .findIndexOnly(cfg.maxNoteLength)(ftsIn, user.account, batch) + ok <- Ok( + Conversions.mkItemListWithTagsFtsPlain(items, batch, limitCapped) ) - ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) } yield ok case _ => @@ -429,17 +432,20 @@ object ItemRoutes { )( settings: OSimpleSearch.Settings, fixQuery: Query.Fix, - itemQuery: ItemQueryString + itemQuery: ItemQueryString, + limitCapped: Boolean ): F[Response[F]] = { import dsl._ def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList = - if (res.indexOnly) Conversions.mkItemListFtsPlain(res.items) - else Conversions.mkItemListFts(res.items) + if (res.indexOnly) + Conversions.mkItemListFtsPlain(res.items, settings.batch, limitCapped) + else Conversions.mkItemListFts(res.items, settings.batch, limitCapped) def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList = - if (res.indexOnly) Conversions.mkItemListWithTagsFtsPlain(res.items) - else Conversions.mkItemListWithTagsFts(res.items) + if (res.indexOnly) + Conversions.mkItemListWithTagsFtsPlain(res.items, settings.batch, limitCapped) + else Conversions.mkItemListWithTagsFts(res.items, settings.batch, limitCapped) backend.simpleSearch .searchByString(settings)(fixQuery, itemQuery) @@ -449,8 +455,8 @@ object ItemRoutes { items.fold( convertFts, convertFtsFull, - Conversions.mkItemList, - Conversions.mkItemListWithTags + els => Conversions.mkItemList(els, settings.batch, limitCapped), + els => Conversions.mkItemListWithTags(els, settings.batch, limitCapped) ) ) case StringSearchResult.FulltextMismatch(TooMany) => diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala index 806691b1..92a66ff1 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala @@ -51,6 +51,7 @@ object ShareSearchRoutes { ).restrictLimitTo( cfg.maxItemPageSize ) + limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize) itemQuery = ItemQueryString(userQuery.query) settings = OSimpleSearch.Settings( batch, @@ -62,7 +63,12 @@ object ShareSearchRoutes { account = share.account fixQuery = Query.Fix(account, Some(share.query.expr), None) _ <- logger.debug(s"Searching in share ${share.id.id}: ${userQuery.query}") - resp <- ItemRoutes.searchItems(backend, dsl)(settings, fixQuery, itemQuery) + resp <- ItemRoutes.searchItems(backend, dsl)( + settings, + fixQuery, + itemQuery, + limitCapped + ) } yield resp } .getOrElseF(NotFound()) diff --git a/modules/webapp/src/main/elm/Data/Items.elm b/modules/webapp/src/main/elm/Data/Items.elm index c545d91a..0cf52354 100644 --- a/modules/webapp/src/main/elm/Data/Items.elm +++ b/modules/webapp/src/main/elm/Data/Items.elm @@ -64,10 +64,10 @@ concat l0 l1 = suff = List.drop 1 l1.groups in - ItemLightList (prev ++ (ng :: suff)) + ItemLightList (prev ++ (ng :: suff)) 0 0 False else - ItemLightList (l0.groups ++ l1.groups) + ItemLightList (l0.groups ++ l1.groups) 0 0 False first : ItemLightList -> Maybe ItemLight @@ -121,7 +121,7 @@ replaceIn origin replacements = |> ItemLightGroup g.name in List.map replaceGroup origin.groups - |> ItemLightList + |> (\els -> ItemLightList els origin.limit origin.offset origin.limitCapped)