From 09d74b7e80cc349cf7bd37f546dafbf8e5972ce0 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 4 Aug 2020 22:45:35 +0200 Subject: [PATCH] Return item notes with search results In order to not make the response very large, a admin can define a limit on how much to return. --- .../docspell/backend/ops/OFulltext.scala | 26 ++++++++++++------- .../docspell/backend/ops/OItemSearch.scala | 16 +++++++----- .../joex/notify/NotifyDueItemsTask.scala | 2 +- .../src/main/resources/docspell-openapi.yml | 4 +++ .../src/main/resources/reference.conf | 6 +++++ .../scala/docspell/restserver/Config.scala | 1 + .../restserver/conv/Conversions.scala | 1 + .../restserver/routes/ItemRoutes.scala | 10 +++---- .../scala/docspell/store/impl/Column.scala | 4 +++ .../scala/docspell/store/queries/QItem.scala | 18 ++++++++++--- nix/module-server.nix | 12 +++++++++ 11 files changed, 75 insertions(+), 25 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index bd1d7622..9bc3f4a6 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -15,20 +15,20 @@ import docspell.store.records.RJob trait OFulltext[F[_]] { - def findItems( + def findItems(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, batch: Batch ): F[Vector[OFulltext.FtsItem]] /** Same as `findItems` but does more queries per item to find all tags. */ - def findItemsWithTags( + def findItemsWithTags(maxNoteLen: Int)( q: Query, fts: OFulltext.FtsInput, batch: Batch ): F[Vector[OFulltext.FtsItemWithTags]] - def findIndexOnly( + def findIndexOnly(maxNoteLen: Int)( fts: OFulltext.FtsInput, account: AccountId, batch: Batch @@ -92,7 +92,7 @@ object OFulltext { else queue.insertIfNew(job) *> joex.notifyAllNodes } yield () - def findIndexOnly( + def findIndexOnly(maxNoteLen: Int)( ftsQ: OFulltext.FtsInput, account: AccountId, batch: Batch @@ -120,7 +120,7 @@ object OFulltext { .transact( QItem.findItemsWithTags( account.collective, - QItem.findSelectedItems(QItem.Query.empty(account), select) + QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select) ) ) .take(batch.limit.toLong) @@ -133,15 +133,23 @@ object OFulltext { } yield res } - def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] = - findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem]) + def findItems( + maxNoteLen: Int + )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] = + findItemsFts( + q, + ftsQ, + batch.first, + itemSearch.findItems(maxNoteLen), + convertFtsData[ListItem] + ) .drop(batch.offset.toLong) .take(batch.limit.toLong) .map({ case (li, fd) => FtsItem(li, fd) }) .compile .toVector - def findItemsWithTags( + def findItemsWithTags(maxNoteLen: Int)( q: Query, ftsQ: FtsInput, batch: Batch @@ -150,7 +158,7 @@ object OFulltext { q, ftsQ, batch.first, - itemSearch.findItemsWithTags, + itemSearch.findItemsWithTags(maxNoteLen), convertFtsData[ListItemWithTags] ) .drop(batch.offset.toLong) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index e4b42b24..44fe2e71 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -17,10 +17,12 @@ import doobie.implicits._ trait OItemSearch[F[_]] { def findItem(id: Ident, collective: Ident): F[Option[ItemData]] - def findItems(q: Query, batch: Batch): F[Vector[ListItem]] + def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] /** Same as `findItems` but does more queries per item to find all tags. */ - def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] + def findItemsWithTags( + maxNoteLen: Int + )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] @@ -97,14 +99,16 @@ object OItemSearch { .transact(QItem.findItem(id)) .map(opt => opt.flatMap(_.filterCollective(collective))) - def findItems(q: Query, batch: Batch): F[Vector[ListItem]] = + def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] = store - .transact(QItem.findItems(q, batch).take(batch.limit.toLong)) + .transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong)) .compile .toVector - def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = { - val search = QItem.findItems(q, batch) + def findItemsWithTags( + maxNoteLen: Int + )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = { + val search = QItem.findItems(q, maxNoteLen: Int, batch) store .transact( QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong) diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index 1eb24a75..b4a59291 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -82,7 +82,7 @@ object NotifyDueItemsTask { ) res <- ctx.store - .transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong)) + .transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong)) .compile .toVector } yield res diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 5ee05848..c5451378 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3860,6 +3860,10 @@ components: type: array items: $ref: "#/components/schemas/Tag" + notes: + description: | + Some prefix of the item notes. + type: string highlighting: description: | Optional contextual information of a search query. Each diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 1142c2cb..c9ec753e 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -24,6 +24,12 @@ docspell.server { # depending on the available resources. max-item-page-size = 200 + # The number of characters to return for each item notes when + # searching. Item notes may be very long, when returning them with + # all the results from a search, they add quite some data to return. + # In order to keep this low, a limit can be defined here. + max-note-length = 180 + # Authentication. auth { diff --git a/modules/restserver/src/main/scala/docspell/restserver/Config.scala b/modules/restserver/src/main/scala/docspell/restserver/Config.scala index b83ee33f..b665e8e7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Config.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Config.scala @@ -16,6 +16,7 @@ case class Config( auth: Login.Config, integrationEndpoint: Config.IntegrationEndpoint, maxItemPageSize: Int, + maxNoteLength: Int, fullTextSearch: Config.FullTextSearch ) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 7c57b5e3..5cf79d9b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -197,6 +197,7 @@ trait Conversions { i.folder.map(mkIdName), i.fileCount, Nil, + i.notes, Nil ) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index 02eabd9c..d94ef314 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -40,7 +40,7 @@ object ItemRoutes { resp <- mask.fullText match { case Some(fq) if cfg.fullTextSearch.enabled => for { - items <- backend.fulltext.findItems( + items <- backend.fulltext.findItems(cfg.maxNoteLength)( query, OFulltext.FtsInput(fq), Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) @@ -49,7 +49,7 @@ object ItemRoutes { } yield ok case _ => for { - items <- backend.itemSearch.findItems( + items <- backend.itemSearch.findItems(cfg.maxNoteLength)( query, Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) ) @@ -67,7 +67,7 @@ object ItemRoutes { resp <- mask.fullText match { case Some(fq) if cfg.fullTextSearch.enabled => for { - items <- backend.fulltext.findItemsWithTags( + items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)( query, OFulltext.FtsInput(fq), Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) @@ -76,7 +76,7 @@ object ItemRoutes { } yield ok case _ => for { - items <- backend.itemSearch.findItemsWithTags( + items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)( query, Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) ) @@ -92,7 +92,7 @@ object ItemRoutes { case q if q.length > 1 => val ftsIn = OFulltext.FtsInput(q) for { - items <- backend.fulltext.findIndexOnly( + items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( ftsIn, user.account, Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala index 134e0afb..de495170 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Column.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Column.scala @@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") { def decrement[A: Put](a: A): Fragment = f ++ fr"=" ++ f ++ fr"- $a" + + def substring(from: Int, many: Int): Fragment = + if (many <= 0 || from < 0) fr"${""}" + else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)" } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index bc6dc7ce..1ce0e976 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -156,7 +156,8 @@ object QItem { corrPerson: Option[IdRef], concPerson: Option[IdRef], concEquip: Option[IdRef], - folder: Option[IdRef] + folder: Option[IdRef], + notes: Option[String] ) case class Query( @@ -228,6 +229,7 @@ object QItem { private def findItemsBase( q: Query, distinct: Boolean, + noteMaxLen: Int, moreCols: Seq[Fragment], ctes: (String, Fragment)* ): Fragment = { @@ -264,6 +266,9 @@ object QItem { EC.name.prefix("e1").f, FC.id.prefix("f1").f, FC.name.prefix("f1").f, + // sql uses 1 for first character + IC.notes.prefix("i").substring(1, noteMaxLen), + // last column is only for sorting q.orderAsc match { case Some(co) => coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f) @@ -307,14 +312,16 @@ object QItem { fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1")) } - def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = { + def findItems( + q: Query, + maxNoteLen: Int, + batch: Batch + ): Stream[ConnectionIO, ListItem] = { val IC = RItem.Columns val PC = RPerson.Columns val OC = ROrganization.Columns val EC = REquipment.Columns - val query = findItemsBase(q, true, Seq.empty) - // inclusive tags are AND-ed val tagSelectsIncl = q.tagsInclude .map(tid => @@ -404,6 +411,7 @@ object QItem { if (batch == Batch.all) Fragment.empty else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}" + val query = findItemsBase(q, true, maxNoteLen, Seq.empty) val frag = query ++ fr"WHERE" ++ cond ++ order ++ limitOffset logger.trace(s"List $batch items: $frag") @@ -413,6 +421,7 @@ object QItem { case class SelectedItem(itemId: Ident, weight: Double) def findSelectedItems( q: Query, + maxNoteLen: Int, items: Set[SelectedItem] ): Stream[ConnectionIO, ListItem] = if (items.isEmpty) Stream.empty @@ -425,6 +434,7 @@ object QItem { val from = findItemsBase( q, true, + maxNoteLen, Seq(fr"tids.weight"), ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")") ) ++ diff --git a/nix/module-server.nix b/nix/module-server.nix index e713b508..b75ad86e 100644 --- a/nix/module-server.nix +++ b/nix/module-server.nix @@ -14,6 +14,7 @@ let app-id = "rest1"; base-url = "http://localhost:7880"; max-item-page-size = 200; + max-note-length = 180; bind = { address = "localhost"; port = 7880; @@ -124,6 +125,17 @@ in { ''; }; + max-note-length = mkOption { + type = types.int; + default = defaults.max-note-length; + description = '' + The number of characters to return for each item notes when + searching. Item notes may be very long, when returning them with + all the results from a search, they add quite some data to return. + In order to keep this low, a limit can be defined here. + ''; + }; + bind = mkOption { type = types.submodule({ options = {