Implement search by query in endpoints

modules/backend/src/main/scala/docspell/backend/BackendApp.scala

@@ -37,6 +37,7 @@ trait BackendApp[F[_]] {
   def userTask: OUserTask[F]
   def folder: OFolder[F]
   def customFields: OCustomFields[F]
+  def simpleSearch: OSimpleSearch[F]
 }
 
 object BackendApp {
@@ -71,6 +72,7 @@ object BackendApp {
       userTaskImpl     <- OUserTask(utStore, queue, joexImpl)
       folderImpl       <- OFolder(store)
       customFieldsImpl <- OCustomFields(store)
+      simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl)
     } yield new BackendApp[F] {
       val login        = loginImpl
       val signup       = signupImpl
@@ -90,6 +92,7 @@ object BackendApp {
       val userTask     = userTaskImpl
       val folder       = folderImpl
       val customFields = customFieldsImpl
+      val simpleSearch = simpleSearchImpl
     }
 
   def apply[F[_]: ConcurrentEffect: ContextShift](

modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala

@@ -1,29 +1,28 @@
 package docspell.backend.ops
 
+import cats.effect.Sync
 import cats.implicits._
 
+import docspell.backend.ops.OSimpleSearch._
 import docspell.common._
+import docspell.query.{ItemQueryParser, ParseFailure}
 import docspell.store.qb.Batch
 import docspell.store.queries.Query
-import docspell.query.{ItemQueryParser, ParseFailure}
-
-import OSimpleSearch._
 import docspell.store.queries.SearchSummary
-import cats.effect.Sync
 
 /** A "porcelain" api on top of OFulltext and OItemSearch. */
 trait OSimpleSearch[F[_]] {
 
   def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
   def searchSummary(
-      settings: Settings
+      useFTS: Boolean
   )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
 
   def searchByString(
       settings: Settings
   )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]]
   def searchSummaryByString(
-      settings: Settings
+      useFTS: Boolean
   )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]]
 
 }
@@ -36,12 +35,6 @@ object OSimpleSearch {
       resolveDetails: Boolean,
       maxNoteLen: Int
   )
-  object Settings {
-    def plain(batch: Batch, useFulltext: Boolean, maxNoteLen: Int): Settings =
-      Settings(batch, useFulltext, false, maxNoteLen)
-    def detailed(batch: Batch, useFulltext: Boolean, maxNoteLen: Int): Settings =
-      Settings(batch, useFulltext, true, maxNoteLen)
-  }
 
   sealed trait Items {
     def fold[A](
@@ -118,18 +111,18 @@ object OSimpleSearch {
         .map(search(settings)(_, None)) //TODO resolve content:xyz expressions
 
     def searchSummaryByString(
-        settings: Settings
+        useFTS: Boolean
     )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] =
       ItemQueryParser
         .parse(q.query)
         .map(iq => Query(fix, Query.QueryExpr(iq)))
-        .map(searchSummary(settings)(_, None)) //TODO resolve content:xyz expressions
+        .map(searchSummary(useFTS)(_, None)) //TODO resolve content:xyz expressions
 
     def searchSummary(
-        settings: Settings
+        useFTS: Boolean
     )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
       fulltextQuery match {
-        case Some(ftq) if settings.useFTS =>
+        case Some(ftq) if useFTS =>
           if (q.isEmpty)
             fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq))
           else

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -1310,16 +1310,16 @@ paths:
                 $ref: "#/components/schemas/BasicResult"
 
 
-  /sec/item/search:
+  /sec/item/searchForm:
     post:
-      tags: [ Item ]
+      tags: [ Item Search ]
       summary: Search for items.
       description: |
         Search for items given a search form. The results are grouped
         by month and are sorted by item date (newest first). Tags and
         attachments are *not* resolved. The results will always
         contain an empty list for item tags and attachments. Use
-        `/searchWithTags` to also retrieve all tags and a list of
+        `/searchFormWithTags` to also retrieve all tags and a list of
         attachments of an item.
 
         The `fulltext` field can be used to restrict the results by
@@ -1328,6 +1328,8 @@ paths:
         The customfields used in the search query are allowed to be
         specified by either field id or field name. The values may
         contain the wildcard `*` at beginning or end.
+
+        **NOTE** This is deprecated in favor for using a search query.
       security:
         - authTokenHeader: []
       requestBody:
@@ -1342,9 +1344,9 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/ItemLightList"
-  /sec/item/searchWithTags:
+  /sec/item/searchFormWithTags:
     post:
-      tags: [ Item ]
+      tags: [ Item Search ]
       summary: Search for items.
       description: |
         Search for items given a search form. The results are grouped
@@ -1355,6 +1357,8 @@ paths:
 
         The `fulltext` field can be used to restrict the results by
         using full-text search in the documents contents.
+
+        **NOTE** This is deprecated in favor for using search query.
       security:
         - authTokenHeader: []
       requestBody:
@@ -1369,9 +1373,60 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/ItemLightList"
+
+  /sec/item/search:
+    get:
+      tags: [ Item Search ]
+      summary: Search for items.
+      description: |
+        Search for items given a search query. The results are grouped
+        by month and are sorted by item date (newest first). Tags and
+        attachments are *not* resolved. The results will always
+        contain an empty list for item tags and attachments. Set
+        `withDetails` to `true` for retrieving all tags and a list of
+        attachments of an item.
+      security:
+        - authTokenHeader: []
+      parameters:
+        - $ref: "#/components/parameters/q"
+        - $ref: "#/components/parameters/limit"
+        - $ref: "#/components/parameters/offset"
+        - $ref: "#/components/parameters/withDetails"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ItemLightList"
+    post:
+      tags: [ Item Search ]
+      summary: Search for items.
+      description: |
+        Search for items given a search query. The results are grouped
+        by month and are sorted by item date (newest first). Tags and
+        attachments are *not* resolved. The results will always
+        contain an empty list for item tags and attachments. Use
+        `withDetails` to also retrieve all tags and a list of
+        attachments of an item.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ItemQuery"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ItemLightList"
+
   /sec/item/searchIndex:
     post:
-      tags: [ Item ]
+      tags: [ Item Search ]
       summary: Search for items using full-text search only.
       description: |
         Search for items by only using the full-text search index.
@@ -1391,7 +1446,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/ItemFtsSearch"
+              $ref: "#/components/schemas/ItemQuery"
       responses:
         200:
           description: Ok
@@ -1400,12 +1455,14 @@ paths:
               schema:
                 $ref: "#/components/schemas/ItemLightList"
 
-  /sec/item/searchStats:
+  /sec/item/searchFormStats:
     post:
-      tags: [ Item ]
+      tags: [ Item Search ]
       summary: Get basic statistics about the data of a search.
       description: |
         Takes a search query and returns a summary about the results.
+
+        **NOTE** This is deprecated in favor of using a search query.
       security:
         - authTokenHeader: []
       requestBody:
@@ -1420,6 +1477,44 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/SearchStats"
+  /sec/item/searchStats:
+    post:
+      tags: [ Item Search ]
+      summary: Get basic statistics about search results.
+      description: |
+        Instead of returning the results of a query, uses it to return
+        a summary.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ItemQuery"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SearchStats"
+    get:
+      tags: [ Item Search ]
+      summary: Get basic statistics about search results.
+      description: |
+        Instead of returning the results of a query, uses it to return
+        a summary.
+      security:
+        - authTokenHeader: []
+      parameters:
+        - $ref: "#/components/parameters/q"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SearchStats"
 
   /sec/item/{id}:
     get:
@@ -3777,22 +3872,12 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/IdName"
-    FolderMember:
+    ItemQuery:
       description: |
-        Information to add or remove a folder member.
+        Query description for a search. Is used for fulltext-only
-      required:
+        searches and combined searches.
-        - userId
-      properties:
-        userId:
-          type: string
-          format: ident
-    ItemFtsSearch:
-      description: |
-        Query description for a full-text only search.
       required:
         - query
-        - offset
-        - limit
       properties:
         offset:
           type: integer
@@ -3804,6 +3889,9 @@ components:
             The maximum number of results to return. Note that this
             limit is a soft limit, there is some hard limit on the
             server, too.
+        withDetails:
+          type: boolean
+          default: false
         query:
           type: string
           description: |
@@ -5547,6 +5635,26 @@ components:
       required: false
       schema:
         type: string
+    limit:
+      name: limit
+      in: query
+      description: A limit for a search query
+      schema:
+        type: integer
+        format: int32
+    offset:
+      name: offset
+      in: query
+      description: A offset into the results for a search query
+      schema:
+        type: integer
+        format: int32
+    withDetails:
+      name: withDetails
+      in: query
+      description: Whether to return details to each item.
+      schema:
+        type: boolean
     name:
       name: name
       in: path

modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala

@@ -28,6 +28,7 @@ object QueryParam {
   object Query       extends OptionalQueryParamDecoderMatcher[String]("q")
   object Limit       extends OptionalQueryParamDecoderMatcher[Int]("limit")
   object Offset      extends OptionalQueryParamDecoderMatcher[Int]("offset")
+  object WithDetails extends OptionalQueryParamDecoderMatcher[Boolean]("withDetails")
 
   object WithFallback extends OptionalQueryParamDecoderMatcher[Boolean]("withFallback")
 }

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -10,9 +10,9 @@ import docspell.backend.auth.AuthToken
 import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue}
 import docspell.backend.ops.OFulltext
 import docspell.backend.ops.OItemSearch.{Batch, Query}
+import docspell.backend.ops.OSimpleSearch
 import docspell.common._
 import docspell.common.syntax.all._
-import docspell.query.ItemQueryParser
 import docspell.restapi.model._
 import docspell.restserver.Config
 import docspell.restserver.conv.Conversions
@@ -49,30 +49,96 @@ object ItemRoutes {
 
       case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
             offset
-          ) =>
+          ) :? QP.WithDetails(detailFlag) =>
-        val query =
+        val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
-          ItemQueryParser.parse(q.getOrElse("")) match {
+          .restrictLimitTo(cfg.maxItemPageSize)
-            case Right(q) =>
+        val itemQuery = ItemQueryString(q)
-              Right(Query(Query.Fix(user.account, None, None), Query.QueryExpr(q)))
+        val settings = OSimpleSearch.Settings(
-            case Left(err) =>
+          batch,
-              Left(err)
+          cfg.fullTextSearch.enabled,
-          }
+          detailFlag.getOrElse(false),
-        val li = limit.getOrElse(cfg.maxItemPageSize)
+          cfg.maxNoteLength
-        val of = offset.getOrElse(0)
-        query match {
-          case Left(err) =>
-            BadRequest(BasicResult(false, err.render))
-          case Right(sq) =>
-            for {
-              items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
-                sq,
-                Batch(of, li).restrictLimitTo(cfg.maxItemPageSize)
         )
-              ok <- Ok(Conversions.mkItemList(items))
+        val fixQuery = Query.Fix(user.account, None, None)
-            } yield ok
+        backend.simpleSearch.searchByString(settings)(fixQuery, itemQuery) match {
+          case Right(results) =>
+            val items = results.map(
+              _.fold(
+                Conversions.mkItemListFts,
+                Conversions.mkItemListWithTagsFts,
+                Conversions.mkItemList,
+                Conversions.mkItemListWithTags
+              )
+            )
+            Ok(items)
+          case Left(fail) =>
+            BadRequest(BasicResult(false, fail.render))
+        }
+
+      case GET -> Root / "searchStats" :? QP.Query(q) =>
+        val itemQuery = ItemQueryString(q)
+        val fixQuery  = Query.Fix(user.account, None, None)
+        backend.simpleSearch
+          .searchSummaryByString(cfg.fullTextSearch.enabled)(fixQuery, itemQuery) match {
+          case Right(summary) =>
+            summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
+          case Left(fail) =>
+            BadRequest(BasicResult(false, fail.render))
         }
 
       case req @ POST -> Root / "search" =>
+        for {
+          userQuery <- req.as[ItemQuery]
+          batch = Batch(
+            userQuery.offset.getOrElse(0),
+            userQuery.limit.getOrElse(cfg.maxItemPageSize)
+          ).restrictLimitTo(
+            cfg.maxItemPageSize
+          )
+          itemQuery = ItemQueryString(userQuery.query)
+          settings = OSimpleSearch.Settings(
+            batch,
+            cfg.fullTextSearch.enabled,
+            userQuery.withDetails.getOrElse(false),
+            cfg.maxNoteLength
+          )
+          fixQuery = Query.Fix(user.account, None, None)
+          resp <- backend.simpleSearch
+            .searchByString(settings)(fixQuery, itemQuery) match {
+            case Right(results) =>
+              val items = results.map(
+                _.fold(
+                  Conversions.mkItemListFts,
+                  Conversions.mkItemListWithTagsFts,
+                  Conversions.mkItemList,
+                  Conversions.mkItemListWithTags
+                )
+              )
+              Ok(items)
+            case Left(fail) =>
+              BadRequest(BasicResult(false, fail.render))
+          }
+        } yield resp
+
+      case req @ POST -> Root / "searchStats" =>
+        for {
+          userQuery <- req.as[ItemQuery]
+          itemQuery = ItemQueryString(userQuery.query)
+          fixQuery  = Query.Fix(user.account, None, None)
+          resp <- backend.simpleSearch
+            .searchSummaryByString(cfg.fullTextSearch.enabled)(
+              fixQuery,
+              itemQuery
+            ) match {
+            case Right(summary) =>
+              summary.flatMap(s => Ok(Conversions.mkSearchStats(s)))
+            case Left(fail) =>
+              BadRequest(BasicResult(false, fail.render))
+          }
+        } yield resp
+
+      //DEPRECATED
+      case req @ POST -> Root / "searchForm" =>
         for {
           mask <- req.as[ItemSearch]
           _    <- logger.ftrace(s"Got search mask: $mask")
@@ -111,7 +177,8 @@ object ItemRoutes {
           }
         } yield resp
 
-      case req @ POST -> Root / "searchWithTags" =>
+      //DEPRECATED
+      case req @ POST -> Root / "searchFormWithTags" =>
         for {
           mask <- req.as[ItemSearch]
           _    <- logger.ftrace(s"Got search mask: $mask")
@@ -151,7 +218,7 @@ object ItemRoutes {
 
       case req @ POST -> Root / "searchIndex" =>
         for {
-          mask <- req.as[ItemFtsSearch]
+          mask <- req.as[ItemQuery]
           resp <- mask.query match {
             case q if q.length > 1 =>
               val ftsIn = OFulltext.FtsInput(q)
@@ -159,7 +226,10 @@ object ItemRoutes {
                 items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
                   ftsIn,
                   user.account,
-                  Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
+                  Batch(
+                    mask.offset.getOrElse(0),
+                    mask.limit.getOrElse(cfg.maxItemPageSize)
+                  ).restrictLimitTo(cfg.maxItemPageSize)
                 )
                 ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items))
               } yield ok
@@ -169,7 +239,8 @@ object ItemRoutes {
           }
         } yield resp
 
-      case req @ POST -> Root / "searchStats" =>
+      //DEPRECATED
+      case req @ POST -> Root / "searchFormStats" =>
         for {
           mask <- req.as[ItemSearch]
           query = Conversions.mkQuery(mask, user.account)
@@ -479,12 +550,12 @@ object ItemRoutes {
     private val itemSearchMonoid: Monoid[ItemSearch] =
       cats.derived.semiauto.monoid
 
-    def unapply(m: ItemSearch): Option[ItemFtsSearch] =
+    def unapply(m: ItemSearch): Option[ItemQuery] =
       m.fullText match {
         case Some(fq) =>
           val me = m.copy(fullText = None, offset = 0, limit = 0)
           if (itemSearchMonoid.empty == me)
-            Some(ItemFtsSearch(m.offset, m.limit, fq))
+            Some(ItemQuery(m.offset.some, m.limit.some, Some(false), fq))
           else None
         case _ =>
           None

modules/webapp/src/main/elm/Api.elm

@@ -156,10 +156,10 @@ import Api.Model.ImapSettings exposing (ImapSettings)
 import Api.Model.ImapSettingsList exposing (ImapSettingsList)
 import Api.Model.InviteResult exposing (InviteResult)
 import Api.Model.ItemDetail exposing (ItemDetail)
-import Api.Model.ItemFtsSearch exposing (ItemFtsSearch)
 import Api.Model.ItemInsights exposing (ItemInsights)
 import Api.Model.ItemLightList exposing (ItemLightList)
 import Api.Model.ItemProposals exposing (ItemProposals)
+import Api.Model.ItemQuery exposing (ItemQuery)
 import Api.Model.ItemSearch exposing (ItemSearch)
 import Api.Model.ItemUploadMeta exposing (ItemUploadMeta)
 import Api.Model.ItemsAndDate exposing (ItemsAndDate)
@@ -1684,14 +1684,14 @@ moveAttachmentBefore flags itemId data receive =
 
 itemIndexSearch :
     Flags
-    -> ItemFtsSearch
+    -> ItemQuery
     -> (Result Http.Error ItemLightList -> msg)
     -> Cmd msg
 itemIndexSearch flags query receive =
     Http2.authPost
         { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchIndex"
         , account = getAccount flags
-        , body = Http.jsonBody (Api.Model.ItemFtsSearch.encode query)
+        , body = Http.jsonBody (Api.Model.ItemQuery.encode query)
         , expect = Http.expectJson receive Api.Model.ItemLightList.decoder
         }
 
@@ -1699,7 +1699,7 @@ itemIndexSearch flags query receive =
 itemSearch : Flags -> ItemSearch -> (Result Http.Error ItemLightList -> msg) -> Cmd msg
 itemSearch flags search receive =
     Http2.authPost
-        { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchWithTags"
+        { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchFormWithTags"
         , account = getAccount flags
         , body = Http.jsonBody (Api.Model.ItemSearch.encode search)
         , expect = Http.expectJson receive Api.Model.ItemLightList.decoder
@@ -1709,7 +1709,7 @@ itemSearch flags search receive =
 itemSearchStats : Flags -> ItemSearch -> (Result Http.Error SearchStats -> msg) -> Cmd msg
 itemSearchStats flags search receive =
     Http2.authPost
-        { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchStats"
+        { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchFormStats"
         , account = getAccount flags
         , body = Http.jsonBody (Api.Model.ItemSearch.encode search)
         , expect = Http.expectJson receive Api.Model.SearchStats.decoder