diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 81328296..80df397c 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -37,6 +37,7 @@ trait BackendApp[F[_]] { def userTask: OUserTask[F] def folder: OFolder[F] def customFields: OCustomFields[F] + def simpleSearch: OSimpleSearch[F] } object BackendApp { @@ -71,6 +72,7 @@ object BackendApp { userTaskImpl <- OUserTask(utStore, queue, joexImpl) folderImpl <- OFolder(store) customFieldsImpl <- OCustomFields(store) + simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl) } yield new BackendApp[F] { val login = loginImpl val signup = signupImpl @@ -90,6 +92,7 @@ object BackendApp { val userTask = userTaskImpl val folder = folderImpl val customFields = customFieldsImpl + val simpleSearch = simpleSearchImpl } def apply[F[_]: ConcurrentEffect: ContextShift]( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala index d4a29a7f..8eb91c7f 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala @@ -1,29 +1,28 @@ package docspell.backend.ops +import cats.effect.Sync import cats.implicits._ +import docspell.backend.ops.OSimpleSearch._ import docspell.common._ +import docspell.query.{ItemQueryParser, ParseFailure} import docspell.store.qb.Batch import docspell.store.queries.Query -import docspell.query.{ItemQueryParser, ParseFailure} - -import OSimpleSearch._ import docspell.store.queries.SearchSummary -import cats.effect.Sync /** A "porcelain" api on top of OFulltext and OItemSearch. */ trait OSimpleSearch[F[_]] { def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items] def searchSummary( - settings: Settings + useFTS: Boolean )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] def searchByString( settings: Settings )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[Items]] def searchSummaryByString( - settings: Settings + useFTS: Boolean )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] } @@ -36,12 +35,6 @@ object OSimpleSearch { resolveDetails: Boolean, maxNoteLen: Int ) - object Settings { - def plain(batch: Batch, useFulltext: Boolean, maxNoteLen: Int): Settings = - Settings(batch, useFulltext, false, maxNoteLen) - def detailed(batch: Batch, useFulltext: Boolean, maxNoteLen: Int): Settings = - Settings(batch, useFulltext, true, maxNoteLen) - } sealed trait Items { def fold[A]( @@ -118,18 +111,18 @@ object OSimpleSearch { .map(search(settings)(_, None)) //TODO resolve content:xyz expressions def searchSummaryByString( - settings: Settings + useFTS: Boolean )(fix: Query.Fix, q: ItemQueryString): Either[ParseFailure, F[SearchSummary]] = ItemQueryParser .parse(q.query) .map(iq => Query(fix, Query.QueryExpr(iq))) - .map(searchSummary(settings)(_, None)) //TODO resolve content:xyz expressions + .map(searchSummary(useFTS)(_, None)) //TODO resolve content:xyz expressions def searchSummary( - settings: Settings + useFTS: Boolean )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = fulltextQuery match { - case Some(ftq) if settings.useFTS => + case Some(ftq) if useFTS => if (q.isEmpty) fts.findIndexOnlySummary(q.fix.account, OFulltext.FtsInput(ftq)) else diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 8623ade1..a56e911c 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1310,16 +1310,16 @@ paths: $ref: "#/components/schemas/BasicResult" - /sec/item/search: + /sec/item/searchForm: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items. description: | Search for items given a search form. The results are grouped by month and are sorted by item date (newest first). Tags and attachments are *not* resolved. The results will always contain an empty list for item tags and attachments. Use - `/searchWithTags` to also retrieve all tags and a list of + `/searchFormWithTags` to also retrieve all tags and a list of attachments of an item. The `fulltext` field can be used to restrict the results by @@ -1328,6 +1328,8 @@ paths: The customfields used in the search query are allowed to be specified by either field id or field name. The values may contain the wildcard `*` at beginning or end. + + **NOTE** This is deprecated in favor for using a search query. security: - authTokenHeader: [] requestBody: @@ -1342,9 +1344,9 @@ paths: application/json: schema: $ref: "#/components/schemas/ItemLightList" - /sec/item/searchWithTags: + /sec/item/searchFormWithTags: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items. description: | Search for items given a search form. The results are grouped @@ -1355,6 +1357,8 @@ paths: The `fulltext` field can be used to restrict the results by using full-text search in the documents contents. + + **NOTE** This is deprecated in favor for using search query. security: - authTokenHeader: [] requestBody: @@ -1369,9 +1373,60 @@ paths: application/json: schema: $ref: "#/components/schemas/ItemLightList" + + /sec/item/search: + get: + tags: [ Item Search ] + summary: Search for items. + description: | + Search for items given a search query. The results are grouped + by month and are sorted by item date (newest first). Tags and + attachments are *not* resolved. The results will always + contain an empty list for item tags and attachments. Set + `withDetails` to `true` for retrieving all tags and a list of + attachments of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/q" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/offset" + - $ref: "#/components/parameters/withDetails" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemLightList" + post: + tags: [ Item Search ] + summary: Search for items. + description: | + Search for items given a search query. The results are grouped + by month and are sorted by item date (newest first). Tags and + attachments are *not* resolved. The results will always + contain an empty list for item tags and attachments. Use + `withDetails` to also retrieve all tags and a list of + attachments of an item. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemQuery" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemLightList" + /sec/item/searchIndex: post: - tags: [ Item ] + tags: [ Item Search ] summary: Search for items using full-text search only. description: | Search for items by only using the full-text search index. @@ -1391,7 +1446,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ItemFtsSearch" + $ref: "#/components/schemas/ItemQuery" responses: 200: description: Ok @@ -1400,12 +1455,14 @@ paths: schema: $ref: "#/components/schemas/ItemLightList" - /sec/item/searchStats: + /sec/item/searchFormStats: post: - tags: [ Item ] + tags: [ Item Search ] summary: Get basic statistics about the data of a search. description: | Takes a search query and returns a summary about the results. + + **NOTE** This is deprecated in favor of using a search query. security: - authTokenHeader: [] requestBody: @@ -1420,6 +1477,44 @@ paths: application/json: schema: $ref: "#/components/schemas/SearchStats" + /sec/item/searchStats: + post: + tags: [ Item Search ] + summary: Get basic statistics about search results. + description: | + Instead of returning the results of a query, uses it to return + a summary. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemQuery" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" + get: + tags: [ Item Search ] + summary: Get basic statistics about search results. + description: | + Instead of returning the results of a query, uses it to return + a summary. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/q" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" /sec/item/{id}: get: @@ -3777,22 +3872,12 @@ components: type: array items: $ref: "#/components/schemas/IdName" - FolderMember: + ItemQuery: description: | - Information to add or remove a folder member. - required: - - userId - properties: - userId: - type: string - format: ident - ItemFtsSearch: - description: | - Query description for a full-text only search. + Query description for a search. Is used for fulltext-only + searches and combined searches. required: - query - - offset - - limit properties: offset: type: integer @@ -3804,6 +3889,9 @@ components: The maximum number of results to return. Note that this limit is a soft limit, there is some hard limit on the server, too. + withDetails: + type: boolean + default: false query: type: string description: | @@ -5547,6 +5635,26 @@ components: required: false schema: type: string + limit: + name: limit + in: query + description: A limit for a search query + schema: + type: integer + format: int32 + offset: + name: offset + in: query + description: A offset into the results for a search query + schema: + type: integer + format: int32 + withDetails: + name: withDetails + in: query + description: Whether to return details to each item. + schema: + type: boolean name: name: name in: path diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala index de4c7090..6907c2c4 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala @@ -25,9 +25,10 @@ object QueryParam { object QueryOpt extends OptionalQueryParamDecoderMatcher[QueryString]("q") - object Query extends OptionalQueryParamDecoderMatcher[String]("q") - object Limit extends OptionalQueryParamDecoderMatcher[Int]("limit") - object Offset extends OptionalQueryParamDecoderMatcher[Int]("offset") + object Query extends OptionalQueryParamDecoderMatcher[String]("q") + object Limit extends OptionalQueryParamDecoderMatcher[Int]("limit") + object Offset extends OptionalQueryParamDecoderMatcher[Int]("offset") + object WithDetails extends OptionalQueryParamDecoderMatcher[Boolean]("withDetails") object WithFallback extends OptionalQueryParamDecoderMatcher[Boolean]("withFallback") } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index e93e4e95..9a0fd9cc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -10,9 +10,9 @@ import docspell.backend.auth.AuthToken import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} import docspell.backend.ops.OFulltext import docspell.backend.ops.OItemSearch.{Batch, Query} +import docspell.backend.ops.OSimpleSearch import docspell.common._ import docspell.common.syntax.all._ -import docspell.query.ItemQueryParser import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -49,30 +49,96 @@ object ItemRoutes { case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset( offset - ) => - val query = - ItemQueryParser.parse(q.getOrElse("")) match { - case Right(q) => - Right(Query(Query.Fix(user.account, None, None), Query.QueryExpr(q))) - case Left(err) => - Left(err) - } - val li = limit.getOrElse(cfg.maxItemPageSize) - val of = offset.getOrElse(0) - query match { - case Left(err) => - BadRequest(BasicResult(false, err.render)) - case Right(sq) => - for { - items <- backend.itemSearch.findItems(cfg.maxNoteLength)( - sq, - Batch(of, li).restrictLimitTo(cfg.maxItemPageSize) + ) :? QP.WithDetails(detailFlag) => + val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize)) + .restrictLimitTo(cfg.maxItemPageSize) + val itemQuery = ItemQueryString(q) + val settings = OSimpleSearch.Settings( + batch, + cfg.fullTextSearch.enabled, + detailFlag.getOrElse(false), + cfg.maxNoteLength + ) + val fixQuery = Query.Fix(user.account, None, None) + backend.simpleSearch.searchByString(settings)(fixQuery, itemQuery) match { + case Right(results) => + val items = results.map( + _.fold( + Conversions.mkItemListFts, + Conversions.mkItemListWithTagsFts, + Conversions.mkItemList, + Conversions.mkItemListWithTags ) - ok <- Ok(Conversions.mkItemList(items)) - } yield ok + ) + Ok(items) + case Left(fail) => + BadRequest(BasicResult(false, fail.render)) + } + + case GET -> Root / "searchStats" :? QP.Query(q) => + val itemQuery = ItemQueryString(q) + val fixQuery = Query.Fix(user.account, None, None) + backend.simpleSearch + .searchSummaryByString(cfg.fullTextSearch.enabled)(fixQuery, itemQuery) match { + case Right(summary) => + summary.flatMap(s => Ok(Conversions.mkSearchStats(s))) + case Left(fail) => + BadRequest(BasicResult(false, fail.render)) } case req @ POST -> Root / "search" => + for { + userQuery <- req.as[ItemQuery] + batch = Batch( + userQuery.offset.getOrElse(0), + userQuery.limit.getOrElse(cfg.maxItemPageSize) + ).restrictLimitTo( + cfg.maxItemPageSize + ) + itemQuery = ItemQueryString(userQuery.query) + settings = OSimpleSearch.Settings( + batch, + cfg.fullTextSearch.enabled, + userQuery.withDetails.getOrElse(false), + cfg.maxNoteLength + ) + fixQuery = Query.Fix(user.account, None, None) + resp <- backend.simpleSearch + .searchByString(settings)(fixQuery, itemQuery) match { + case Right(results) => + val items = results.map( + _.fold( + Conversions.mkItemListFts, + Conversions.mkItemListWithTagsFts, + Conversions.mkItemList, + Conversions.mkItemListWithTags + ) + ) + Ok(items) + case Left(fail) => + BadRequest(BasicResult(false, fail.render)) + } + } yield resp + + case req @ POST -> Root / "searchStats" => + for { + userQuery <- req.as[ItemQuery] + itemQuery = ItemQueryString(userQuery.query) + fixQuery = Query.Fix(user.account, None, None) + resp <- backend.simpleSearch + .searchSummaryByString(cfg.fullTextSearch.enabled)( + fixQuery, + itemQuery + ) match { + case Right(summary) => + summary.flatMap(s => Ok(Conversions.mkSearchStats(s))) + case Left(fail) => + BadRequest(BasicResult(false, fail.render)) + } + } yield resp + + //DEPRECATED + case req @ POST -> Root / "searchForm" => for { mask <- req.as[ItemSearch] _ <- logger.ftrace(s"Got search mask: $mask") @@ -111,7 +177,8 @@ object ItemRoutes { } } yield resp - case req @ POST -> Root / "searchWithTags" => + //DEPRECATED + case req @ POST -> Root / "searchFormWithTags" => for { mask <- req.as[ItemSearch] _ <- logger.ftrace(s"Got search mask: $mask") @@ -151,7 +218,7 @@ object ItemRoutes { case req @ POST -> Root / "searchIndex" => for { - mask <- req.as[ItemFtsSearch] + mask <- req.as[ItemQuery] resp <- mask.query match { case q if q.length > 1 => val ftsIn = OFulltext.FtsInput(q) @@ -159,7 +226,10 @@ object ItemRoutes { items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( ftsIn, user.account, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) + Batch( + mask.offset.getOrElse(0), + mask.limit.getOrElse(cfg.maxItemPageSize) + ).restrictLimitTo(cfg.maxItemPageSize) ) ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) } yield ok @@ -169,7 +239,8 @@ object ItemRoutes { } } yield resp - case req @ POST -> Root / "searchStats" => + //DEPRECATED + case req @ POST -> Root / "searchFormStats" => for { mask <- req.as[ItemSearch] query = Conversions.mkQuery(mask, user.account) @@ -479,12 +550,12 @@ object ItemRoutes { private val itemSearchMonoid: Monoid[ItemSearch] = cats.derived.semiauto.monoid - def unapply(m: ItemSearch): Option[ItemFtsSearch] = + def unapply(m: ItemSearch): Option[ItemQuery] = m.fullText match { case Some(fq) => val me = m.copy(fullText = None, offset = 0, limit = 0) if (itemSearchMonoid.empty == me) - Some(ItemFtsSearch(m.offset, m.limit, fq)) + Some(ItemQuery(m.offset.some, m.limit.some, Some(false), fq)) else None case _ => None diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index efc1d551..1064da36 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -156,10 +156,10 @@ import Api.Model.ImapSettings exposing (ImapSettings) import Api.Model.ImapSettingsList exposing (ImapSettingsList) import Api.Model.InviteResult exposing (InviteResult) import Api.Model.ItemDetail exposing (ItemDetail) -import Api.Model.ItemFtsSearch exposing (ItemFtsSearch) import Api.Model.ItemInsights exposing (ItemInsights) import Api.Model.ItemLightList exposing (ItemLightList) import Api.Model.ItemProposals exposing (ItemProposals) +import Api.Model.ItemQuery exposing (ItemQuery) import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.ItemUploadMeta exposing (ItemUploadMeta) import Api.Model.ItemsAndDate exposing (ItemsAndDate) @@ -1684,14 +1684,14 @@ moveAttachmentBefore flags itemId data receive = itemIndexSearch : Flags - -> ItemFtsSearch + -> ItemQuery -> (Result Http.Error ItemLightList -> msg) -> Cmd msg itemIndexSearch flags query receive = Http2.authPost { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchIndex" , account = getAccount flags - , body = Http.jsonBody (Api.Model.ItemFtsSearch.encode query) + , body = Http.jsonBody (Api.Model.ItemQuery.encode query) , expect = Http.expectJson receive Api.Model.ItemLightList.decoder } @@ -1699,7 +1699,7 @@ itemIndexSearch flags query receive = itemSearch : Flags -> ItemSearch -> (Result Http.Error ItemLightList -> msg) -> Cmd msg itemSearch flags search receive = Http2.authPost - { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchWithTags" + { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchFormWithTags" , account = getAccount flags , body = Http.jsonBody (Api.Model.ItemSearch.encode search) , expect = Http.expectJson receive Api.Model.ItemLightList.decoder @@ -1709,7 +1709,7 @@ itemSearch flags search receive = itemSearchStats : Flags -> ItemSearch -> (Result Http.Error SearchStats -> msg) -> Cmd msg itemSearchStats flags search receive = Http2.authPost - { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchStats" + { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchFormStats" , account = getAccount flags , body = Http.jsonBody (Api.Model.ItemSearch.encode search) , expect = Http.expectJson receive Api.Model.SearchStats.decoder