Return item notes with search results

In order to not make the response very large, a admin can define a
limit on how much to return.

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -15,20 +15,20 @@ import docspell.store.records.RJob
 
 trait OFulltext[F[_]] {
 
-  def findItems(
+  def findItems(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
       batch: Batch
   ): F[Vector[OFulltext.FtsItem]]
 
   /** Same as `findItems` but does more queries per item to find all tags. */
-  def findItemsWithTags(
+  def findItemsWithTags(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
       batch: Batch
   ): F[Vector[OFulltext.FtsItemWithTags]]
 
-  def findIndexOnly(
+  def findIndexOnly(maxNoteLen: Int)(
       fts: OFulltext.FtsInput,
       account: AccountId,
       batch: Batch
@@ -92,7 +92,7 @@ object OFulltext {
             else queue.insertIfNew(job) *> joex.notifyAllNodes
         } yield ()
 
-      def findIndexOnly(
+      def findIndexOnly(maxNoteLen: Int)(
           ftsQ: OFulltext.FtsInput,
           account: AccountId,
           batch: Batch
@@ -120,7 +120,7 @@ object OFulltext {
               .transact(
                 QItem.findItemsWithTags(
                   account.collective,
-                  QItem.findSelectedItems(QItem.Query.empty(account), select)
+                  QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select)
                 )
               )
               .take(batch.limit.toLong)
@@ -133,15 +133,23 @@ object OFulltext {
         } yield res
       }
 
-      def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
+      def findItems(
-        findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem])
+          maxNoteLen: Int
+      )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
+        findItemsFts(
+          q,
+          ftsQ,
+          batch.first,
+          itemSearch.findItems(maxNoteLen),
+          convertFtsData[ListItem]
+        )
           .drop(batch.offset.toLong)
           .take(batch.limit.toLong)
           .map({ case (li, fd) => FtsItem(li, fd) })
           .compile
           .toVector
 
-      def findItemsWithTags(
+      def findItemsWithTags(maxNoteLen: Int)(
           q: Query,
           ftsQ: FtsInput,
           batch: Batch
@@ -150,7 +158,7 @@ object OFulltext {
           q,
           ftsQ,
           batch.first,
-          itemSearch.findItemsWithTags,
+          itemSearch.findItemsWithTags(maxNoteLen),
           convertFtsData[ListItemWithTags]
         )
           .drop(batch.offset.toLong)

modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala

@@ -17,10 +17,12 @@ import doobie.implicits._
 trait OItemSearch[F[_]] {
   def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
 
-  def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
+  def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
 
   /** Same as `findItems` but does more queries per item to find all tags. */
-  def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
+  def findItemsWithTags(
+      maxNoteLen: Int
+  )(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
 
   def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
 
@@ -97,14 +99,16 @@ object OItemSearch {
           .transact(QItem.findItem(id))
           .map(opt => opt.flatMap(_.filterCollective(collective)))
 
-      def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
+      def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
         store
-          .transact(QItem.findItems(q, batch).take(batch.limit.toLong))
+          .transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
           .compile
           .toVector
 
-      def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
+      def findItemsWithTags(
-        val search = QItem.findItems(q, batch)
+          maxNoteLen: Int
+      )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
+        val search = QItem.findItems(q, maxNoteLen: Int, batch)
         store
           .transact(
             QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)

modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala

@@ -82,7 +82,7 @@ object NotifyDueItemsTask {
           )
       res <-
         ctx.store
-          .transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong))
+          .transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
           .compile
           .toVector
     } yield res

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -3860,6 +3860,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/Tag"
+        notes:
+          description: |
+            Some prefix of the item notes.
+          type: string
         highlighting:
           description: |
             Optional contextual information of a search query. Each

modules/restserver/src/main/resources/reference.conf

@@ -24,6 +24,12 @@ docspell.server {
   # depending on the available resources.
   max-item-page-size = 200
 
+  # The number of characters to return for each item notes when
+  # searching. Item notes may be very long, when returning them with
+  # all the results from a search, they add quite some data to return.
+  # In order to keep this low, a limit can be defined here.
+  max-note-length = 180
+
   # Authentication.
   auth {
 

modules/restserver/src/main/scala/docspell/restserver/Config.scala

@@ -16,6 +16,7 @@ case class Config(
     auth: Login.Config,
     integrationEndpoint: Config.IntegrationEndpoint,
     maxItemPageSize: Int,
+    maxNoteLength: Int,
     fullTextSearch: Config.FullTextSearch
 )
 

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -197,6 +197,7 @@ trait Conversions {
       i.folder.map(mkIdName),
       i.fileCount,
       Nil,
+      i.notes,
       Nil
     )
 

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -40,7 +40,7 @@ object ItemRoutes {
           resp <- mask.fullText match {
             case Some(fq) if cfg.fullTextSearch.enabled =>
               for {
-                items <- backend.fulltext.findItems(
+                items <- backend.fulltext.findItems(cfg.maxNoteLength)(
                   query,
                   OFulltext.FtsInput(fq),
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -49,7 +49,7 @@ object ItemRoutes {
               } yield ok
             case _ =>
               for {
-                items <- backend.itemSearch.findItems(
+                items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
                   query,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
                 )
@@ -67,7 +67,7 @@ object ItemRoutes {
           resp <- mask.fullText match {
             case Some(fq) if cfg.fullTextSearch.enabled =>
               for {
-                items <- backend.fulltext.findItemsWithTags(
+                items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)(
                   query,
                   OFulltext.FtsInput(fq),
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -76,7 +76,7 @@ object ItemRoutes {
               } yield ok
             case _ =>
               for {
-                items <- backend.itemSearch.findItemsWithTags(
+                items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)(
                   query,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
                 )
@@ -92,7 +92,7 @@ object ItemRoutes {
             case q if q.length > 1 =>
               val ftsIn = OFulltext.FtsInput(q)
               for {
-                items <- backend.fulltext.findIndexOnly(
+                items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
                   ftsIn,
                   user.account,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)

modules/store/src/main/scala/docspell/store/impl/Column.scala

@@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
 
   def decrement[A: Put](a: A): Fragment =
     f ++ fr"=" ++ f ++ fr"- $a"
+
+  def substring(from: Int, many: Int): Fragment =
+    if (many <= 0 || from < 0) fr"${""}"
+    else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)"
 }

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -156,7 +156,8 @@ object QItem {
       corrPerson: Option[IdRef],
       concPerson: Option[IdRef],
       concEquip: Option[IdRef],
-      folder: Option[IdRef]
+      folder: Option[IdRef],
+      notes: Option[String]
   )
 
   case class Query(
@@ -228,6 +229,7 @@ object QItem {
   private def findItemsBase(
       q: Query,
       distinct: Boolean,
+      noteMaxLen: Int,
       moreCols: Seq[Fragment],
       ctes: (String, Fragment)*
   ): Fragment = {
@@ -264,6 +266,9 @@ object QItem {
         EC.name.prefix("e1").f,
         FC.id.prefix("f1").f,
         FC.name.prefix("f1").f,
+        // sql uses 1 for first character
+        IC.notes.prefix("i").substring(1, noteMaxLen),
+        // last column is only for sorting
         q.orderAsc match {
           case Some(co) =>
             coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
@@ -307,14 +312,16 @@ object QItem {
       fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1"))
   }
 
-  def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = {
+  def findItems(
+      q: Query,
+      maxNoteLen: Int,
+      batch: Batch
+  ): Stream[ConnectionIO, ListItem] = {
     val IC = RItem.Columns
     val PC = RPerson.Columns
     val OC = ROrganization.Columns
     val EC = REquipment.Columns
 
-    val query = findItemsBase(q, true, Seq.empty)
-
     // inclusive tags are AND-ed
     val tagSelectsIncl = q.tagsInclude
       .map(tid =>
@@ -404,6 +411,7 @@ object QItem {
       if (batch == Batch.all) Fragment.empty
       else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
 
+    val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
     val frag =
       query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
     logger.trace(s"List $batch items: $frag")
@@ -413,6 +421,7 @@ object QItem {
   case class SelectedItem(itemId: Ident, weight: Double)
   def findSelectedItems(
       q: Query,
+      maxNoteLen: Int,
       items: Set[SelectedItem]
   ): Stream[ConnectionIO, ListItem] =
     if (items.isEmpty) Stream.empty
@@ -425,6 +434,7 @@ object QItem {
       val from = findItemsBase(
         q,
         true,
+        maxNoteLen,
         Seq(fr"tids.weight"),
         ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
       ) ++

nix/module-server.nix

@@ -14,6 +14,7 @@ let
     app-id = "rest1";
     base-url = "http://localhost:7880";
     max-item-page-size = 200;
+    max-note-length = 180;
     bind = {
       address = "localhost";
       port = 7880;
@@ -124,6 +125,17 @@ in {
         '';
       };
 
+      max-note-length = mkOption {
+        type = types.int;
+        default = defaults.max-note-length;
+        description = ''
+          The number of characters to return for each item notes when
+          searching. Item notes may be very long, when returning them with
+          all the results from a search, they add quite some data to return.
+          In order to keep this low, a limit can be defined here.
+        '';
+      };
+
       bind = mkOption {
         type = types.submodule({
           options = {