Merge pull request #1574 from eikek/fix/search-limit-cap

Include limit-capped flag with search response

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -8591,11 +8591,33 @@ components:
         A list of item details.
       required:
         - groups
+        - limit
+        - offset
+        - limitCapped
       properties:
         groups:
           type: array
           items:
             $ref: "#/components/schemas/ItemLightGroup"
+        limit:
+          type: integer
+          format: int32
+          description: |
+            Returns the `limit` value as used for this search. This
+            can deviate from the requested limit, if it exceeds the
+            server defined maximum. See `limitCapped`.
+        offset:
+          type: integer
+          format: int32
+          description: |
+            The `offset` value used for this search.
+        limitCapped:
+          type: boolean
+          description: |
+            The server defines a maximum `limit` value. If the
+            requested `limit` exceeds the server defined one, this
+            flag is set to true. The limit used for the query is
+            returned with this response.
     ItemLightGroup:
       description: |
         A group of items.

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -23,6 +23,7 @@ import docspell.ftsclient.FtsResult
 import docspell.restapi.model._
 import docspell.restserver.conv.Conversions._
 import docspell.restserver.http4s.ContentDisposition
+import docspell.store.qb.Batch
 import docspell.store.queries.{AttachmentLight => QAttachmentLight, IdRefCount}
 import docspell.store.records._
 import docspell.store.{AddResult, UpdateResult}
@@ -171,7 +172,11 @@ trait Conversions {
   def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue =
     OItemSearch.CustomValue(v.field, v.value)
 
-  def mkItemList(v: Vector[OItemSearch.ListItem]): ItemLightList = {
+  def mkItemList(
+      v: Vector[OItemSearch.ListItem],
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList = {
     val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7))
 
     def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup =
@@ -179,10 +184,14 @@ trait Conversions {
 
     val gs =
       groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
-    ItemLightList(gs)
+    ItemLightList(gs, batch.limit, batch.offset, capped)
   }
 
-  def mkItemListFts(v: Vector[OFulltext.FtsItem]): ItemLightList = {
+  def mkItemListFts(
+      v: Vector[OFulltext.FtsItem],
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList = {
     val groups = v.groupBy(item => item.item.date.toUtcDate.toString.substring(0, 7))
 
     def mkGroup(g: (String, Vector[OFulltext.FtsItem])): ItemLightGroup =
@@ -190,10 +199,14 @@ trait Conversions {
 
     val gs =
       groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
-    ItemLightList(gs)
+    ItemLightList(gs, batch.limit, batch.offset, capped)
   }
 
-  def mkItemListWithTags(v: Vector[OItemSearch.ListItemWithTags]): ItemLightList = {
+  def mkItemListWithTags(
+      v: Vector[OItemSearch.ListItemWithTags],
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList = {
     val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
 
     def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup =
@@ -201,10 +214,14 @@ trait Conversions {
 
     val gs =
       groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
-    ItemLightList(gs)
+    ItemLightList(gs, batch.limit, batch.offset, capped)
   }
 
-  def mkItemListWithTagsFts(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList = {
+  def mkItemListWithTagsFts(
+      v: Vector[OFulltext.FtsItemWithTags],
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList = {
     val groups = v.groupBy(ti => ti.item.item.date.toUtcDate.toString.substring(0, 7))
 
     def mkGroup(g: (String, Vector[OFulltext.FtsItemWithTags])): ItemLightGroup =
@@ -212,16 +229,36 @@ trait Conversions {
 
     val gs =
       groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
-    ItemLightList(gs)
+    ItemLightList(gs, batch.limit, batch.offset, capped)
   }
 
-  def mkItemListWithTagsFtsPlain(v: Vector[OFulltext.FtsItemWithTags]): ItemLightList =
+  def mkItemListWithTagsFtsPlain(
-    if (v.isEmpty) ItemLightList(Nil)
+      v: Vector[OFulltext.FtsItemWithTags],
-    else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)))
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList =
+    if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
+    else
+      ItemLightList(
+        List(ItemLightGroup("Results", v.map(mkItemLightWithTags).toList)),
+        batch.limit,
+        batch.offset,
+        capped
+      )
 
-  def mkItemListFtsPlain(v: Vector[OFulltext.FtsItem]): ItemLightList =
+  def mkItemListFtsPlain(
-    if (v.isEmpty) ItemLightList(Nil)
+      v: Vector[OFulltext.FtsItem],
-    else ItemLightList(List(ItemLightGroup("Results", v.map(mkItemLight).toList)))
+      batch: Batch,
+      capped: Boolean
+  ): ItemLightList =
+    if (v.isEmpty) ItemLightList(Nil, batch.limit, batch.offset, capped)
+    else
+      ItemLightList(
+        List(ItemLightGroup("Results", v.map(mkItemLight).toList)),
+        batch.limit,
+        batch.offset,
+        capped
+      )
 
   def mkItemLight(i: OItemSearch.ListItem): ItemLight =
     ItemLight(

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -50,6 +50,7 @@ object ItemRoutes {
           ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
         val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
           .restrictLimitTo(cfg.maxItemPageSize)
+        val limitCapped = limit.exists(_ > cfg.maxItemPageSize)
         val itemQuery = ItemQueryString(q)
         val settings = OSimpleSearch.Settings(
           batch,
@@ -59,7 +60,7 @@ object ItemRoutes {
           searchMode.getOrElse(SearchMode.Normal)
         )
         val fixQuery = Query.Fix(user.account, None, None)
-        searchItems(backend, dsl)(settings, fixQuery, itemQuery)
+        searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
 
       case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
         val itemQuery = ItemQueryString(q)
@@ -79,6 +80,7 @@ object ItemRoutes {
           ).restrictLimitTo(
             cfg.maxItemPageSize
           )
+          limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
           itemQuery = ItemQueryString(userQuery.query)
           settings = OSimpleSearch.Settings(
             batch,
@@ -88,7 +90,7 @@ object ItemRoutes {
             searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
           )
           fixQuery = Query.Fix(user.account, None, None)
-          resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery)
+          resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
         } yield resp
 
       case req @ POST -> Root / "searchStats" =>
@@ -106,19 +108,20 @@ object ItemRoutes {
       case req @ POST -> Root / "searchIndex" =>
         for {
           mask <- req.as[ItemQuery]
+          limitCapped = mask.limit.exists(_ > cfg.maxItemPageSize)
           resp <- mask.query match {
             case q if q.length > 1 =>
               val ftsIn = OFulltext.FtsInput(q)
-              for {
+              val batch = Batch(
-                items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
-                  ftsIn,
-                  user.account,
-                  Batch(
                 mask.offset.getOrElse(0),
                 mask.limit.getOrElse(cfg.maxItemPageSize)
               ).restrictLimitTo(cfg.maxItemPageSize)
+              for {
+                items <- backend.fulltext
+                  .findIndexOnly(cfg.maxNoteLength)(ftsIn, user.account, batch)
+                ok <- Ok(
+                  Conversions.mkItemListWithTagsFtsPlain(items, batch, limitCapped)
                 )
-                ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items))
               } yield ok
 
             case _ =>
@@ -429,17 +432,20 @@ object ItemRoutes {
   )(
       settings: OSimpleSearch.Settings,
       fixQuery: Query.Fix,
-      itemQuery: ItemQueryString
+      itemQuery: ItemQueryString,
+      limitCapped: Boolean
   ): F[Response[F]] = {
     import dsl._
 
     def convertFts(res: OSimpleSearch.Items.FtsItems): ItemLightList =
-      if (res.indexOnly) Conversions.mkItemListFtsPlain(res.items)
+      if (res.indexOnly)
-      else Conversions.mkItemListFts(res.items)
+        Conversions.mkItemListFtsPlain(res.items, settings.batch, limitCapped)
+      else Conversions.mkItemListFts(res.items, settings.batch, limitCapped)
 
     def convertFtsFull(res: OSimpleSearch.Items.FtsItemsFull): ItemLightList =
-      if (res.indexOnly) Conversions.mkItemListWithTagsFtsPlain(res.items)
+      if (res.indexOnly)
-      else Conversions.mkItemListWithTagsFts(res.items)
+        Conversions.mkItemListWithTagsFtsPlain(res.items, settings.batch, limitCapped)
+      else Conversions.mkItemListWithTagsFts(res.items, settings.batch, limitCapped)
 
     backend.simpleSearch
       .searchByString(settings)(fixQuery, itemQuery)
@@ -449,8 +455,8 @@ object ItemRoutes {
             items.fold(
               convertFts,
               convertFtsFull,
-              Conversions.mkItemList,
+              els => Conversions.mkItemList(els, settings.batch, limitCapped),
-              Conversions.mkItemListWithTags
+              els => Conversions.mkItemListWithTags(els, settings.batch, limitCapped)
             )
           )
         case StringSearchResult.FulltextMismatch(TooMany) =>

modules/restserver/src/main/scala/docspell/restserver/routes/ShareSearchRoutes.scala

@@ -51,6 +51,7 @@ object ShareSearchRoutes {
               ).restrictLimitTo(
                 cfg.maxItemPageSize
               )
+              limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
               itemQuery = ItemQueryString(userQuery.query)
               settings = OSimpleSearch.Settings(
                 batch,
@@ -62,7 +63,12 @@ object ShareSearchRoutes {
               account = share.account
               fixQuery = Query.Fix(account, Some(share.query.expr), None)
               _ <- logger.debug(s"Searching in share ${share.id.id}: ${userQuery.query}")
-              resp <- ItemRoutes.searchItems(backend, dsl)(settings, fixQuery, itemQuery)
+              resp <- ItemRoutes.searchItems(backend, dsl)(
+                settings,
+                fixQuery,
+                itemQuery,
+                limitCapped
+              )
             } yield resp
           }
           .getOrElseF(NotFound())

modules/webapp/src/main/elm/Data/Items.elm

@@ -64,10 +64,10 @@ concat l0 l1 =
                     suff =
                         List.drop 1 l1.groups
                 in
-                ItemLightList (prev ++ (ng :: suff))
+                ItemLightList (prev ++ (ng :: suff)) 0 0 False
 
             else
-                ItemLightList (l0.groups ++ l1.groups)
+                ItemLightList (l0.groups ++ l1.groups) 0 0 False
 
 
 first : ItemLightList -> Maybe ItemLight
@@ -121,7 +121,7 @@ replaceIn origin replacements =
                 |> ItemLightGroup g.name
     in
     List.map replaceGroup origin.groups
-        |> ItemLightList
+        |> (\els -> ItemLightList els origin.limit origin.offset origin.limitCapped)