Merge pull request #211 from eikek/notes-in-listitem

Notes in listitem

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -15,20 +15,20 @@ import docspell.store.records.RJob
 
 trait OFulltext[F[_]] {
 
-  def findItems(
+  def findItems(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
       batch: Batch
   ): F[Vector[OFulltext.FtsItem]]
 
   /** Same as `findItems` but does more queries per item to find all tags. */
-  def findItemsWithTags(
+  def findItemsWithTags(maxNoteLen: Int)(
       q: Query,
       fts: OFulltext.FtsInput,
       batch: Batch
   ): F[Vector[OFulltext.FtsItemWithTags]]
 
-  def findIndexOnly(
+  def findIndexOnly(maxNoteLen: Int)(
       fts: OFulltext.FtsInput,
       account: AccountId,
       batch: Batch
@@ -92,7 +92,7 @@ object OFulltext {
             else queue.insertIfNew(job) *> joex.notifyAllNodes
         } yield ()
 
-      def findIndexOnly(
+      def findIndexOnly(maxNoteLen: Int)(
           ftsQ: OFulltext.FtsInput,
           account: AccountId,
           batch: Batch
@@ -120,7 +120,7 @@ object OFulltext {
               .transact(
                 QItem.findItemsWithTags(
                   account.collective,
-                  QItem.findSelectedItems(QItem.Query.empty(account), select)
+                  QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select)
                 )
               )
               .take(batch.limit.toLong)
@@ -133,15 +133,23 @@ object OFulltext {
         } yield res
       }
 
-      def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
+      def findItems(
-        findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem])
+          maxNoteLen: Int
+      )(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
+        findItemsFts(
+          q,
+          ftsQ,
+          batch.first,
+          itemSearch.findItems(maxNoteLen),
+          convertFtsData[ListItem]
+        )
           .drop(batch.offset.toLong)
           .take(batch.limit.toLong)
           .map({ case (li, fd) => FtsItem(li, fd) })
           .compile
           .toVector
 
-      def findItemsWithTags(
+      def findItemsWithTags(maxNoteLen: Int)(
           q: Query,
           ftsQ: FtsInput,
           batch: Batch
@@ -150,7 +158,7 @@ object OFulltext {
           q,
           ftsQ,
           batch.first,
-          itemSearch.findItemsWithTags,
+          itemSearch.findItemsWithTags(maxNoteLen),
           convertFtsData[ListItemWithTags]
         )
           .drop(batch.offset.toLong)

modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala

@@ -17,10 +17,12 @@ import doobie.implicits._
 trait OItemSearch[F[_]] {
   def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
 
-  def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
+  def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
 
   /** Same as `findItems` but does more queries per item to find all tags. */
-  def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
+  def findItemsWithTags(
+      maxNoteLen: Int
+  )(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
 
   def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
 
@@ -97,14 +99,16 @@ object OItemSearch {
           .transact(QItem.findItem(id))
           .map(opt => opt.flatMap(_.filterCollective(collective)))
 
-      def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
+      def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
         store
-          .transact(QItem.findItems(q, batch).take(batch.limit.toLong))
+          .transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
           .compile
           .toVector
 
-      def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
+      def findItemsWithTags(
-        val search = QItem.findItems(q, batch)
+          maxNoteLen: Int
+      )(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
+        val search = QItem.findItems(q, maxNoteLen: Int, batch)
         store
           .transact(
             QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)

modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala

@@ -82,7 +82,7 @@ object NotifyDueItemsTask {
           )
       res <-
         ctx.store
-          .transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong))
+          .transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
           .compile
           .toVector
     } yield res

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -3860,6 +3860,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/Tag"
+        notes:
+          description: |
+            Some prefix of the item notes.
+          type: string
         highlighting:
           description: |
             Optional contextual information of a search query. Each

modules/restserver/src/main/resources/reference.conf

@@ -24,6 +24,12 @@ docspell.server {
   # depending on the available resources.
   max-item-page-size = 200
 
+  # The number of characters to return for each item notes when
+  # searching. Item notes may be very long, when returning them with
+  # all the results from a search, they add quite some data to return.
+  # In order to keep this low, a limit can be defined here.
+  max-note-length = 180
+
   # Authentication.
   auth {
 

modules/restserver/src/main/scala/docspell/restserver/Config.scala

@@ -16,6 +16,7 @@ case class Config(
     auth: Login.Config,
     integrationEndpoint: Config.IntegrationEndpoint,
     maxItemPageSize: Int,
+    maxNoteLength: Int,
     fullTextSearch: Config.FullTextSearch
 )
 

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -197,6 +197,7 @@ trait Conversions {
       i.folder.map(mkIdName),
       i.fileCount,
       Nil,
+      i.notes,
       Nil
     )
 

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -40,7 +40,7 @@ object ItemRoutes {
           resp <- mask.fullText match {
             case Some(fq) if cfg.fullTextSearch.enabled =>
               for {
-                items <- backend.fulltext.findItems(
+                items <- backend.fulltext.findItems(cfg.maxNoteLength)(
                   query,
                   OFulltext.FtsInput(fq),
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -49,7 +49,7 @@ object ItemRoutes {
               } yield ok
             case _ =>
               for {
-                items <- backend.itemSearch.findItems(
+                items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
                   query,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
                 )
@@ -67,7 +67,7 @@ object ItemRoutes {
           resp <- mask.fullText match {
             case Some(fq) if cfg.fullTextSearch.enabled =>
               for {
-                items <- backend.fulltext.findItemsWithTags(
+                items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)(
                   query,
                   OFulltext.FtsInput(fq),
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -76,7 +76,7 @@ object ItemRoutes {
               } yield ok
             case _ =>
               for {
-                items <- backend.itemSearch.findItemsWithTags(
+                items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)(
                   query,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
                 )
@@ -92,7 +92,7 @@ object ItemRoutes {
             case q if q.length > 1 =>
               val ftsIn = OFulltext.FtsInput(q)
               for {
-                items <- backend.fulltext.findIndexOnly(
+                items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
                   ftsIn,
                   user.account,
                   Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)

modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala

@@ -15,7 +15,9 @@ case class Flags(
     signupMode: SignupConfig.Mode,
     docspellAssetPath: String,
     integrationEnabled: Boolean,
-    fullTextSearchEnabled: Boolean
+    fullTextSearchEnabled: Boolean,
+    maxPageSize: Int,
+    maxNoteLength: Int
 )
 
 object Flags {
@@ -26,7 +28,9 @@ object Flags {
       cfg.backend.signup.mode,
       s"/app/assets/docspell-webapp/${BuildInfo.version}",
       cfg.integrationEndpoint.enabled,
-      cfg.fullTextSearch.enabled
+      cfg.fullTextSearch.enabled,
+      cfg.maxItemPageSize,
+      cfg.maxNoteLength
     )
 
   implicit val jsonEncoder: Encoder[Flags] =

modules/store/src/main/scala/docspell/store/impl/Column.scala

@@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
 
   def decrement[A: Put](a: A): Fragment =
     f ++ fr"=" ++ f ++ fr"- $a"
+
+  def substring(from: Int, many: Int): Fragment =
+    if (many <= 0 || from < 0) fr"${""}"
+    else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)"
 }

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -156,7 +156,8 @@ object QItem {
       corrPerson: Option[IdRef],
       concPerson: Option[IdRef],
       concEquip: Option[IdRef],
-      folder: Option[IdRef]
+      folder: Option[IdRef],
+      notes: Option[String]
   )
 
   case class Query(
@@ -228,6 +229,7 @@ object QItem {
   private def findItemsBase(
       q: Query,
       distinct: Boolean,
+      noteMaxLen: Int,
       moreCols: Seq[Fragment],
       ctes: (String, Fragment)*
   ): Fragment = {
@@ -264,6 +266,9 @@ object QItem {
         EC.name.prefix("e1").f,
         FC.id.prefix("f1").f,
         FC.name.prefix("f1").f,
+        // sql uses 1 for first character
+        IC.notes.prefix("i").substring(1, noteMaxLen),
+        // last column is only for sorting
         q.orderAsc match {
           case Some(co) =>
             coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
@@ -307,14 +312,16 @@ object QItem {
       fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1"))
   }
 
-  def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = {
+  def findItems(
+      q: Query,
+      maxNoteLen: Int,
+      batch: Batch
+  ): Stream[ConnectionIO, ListItem] = {
     val IC = RItem.Columns
     val PC = RPerson.Columns
     val OC = ROrganization.Columns
     val EC = REquipment.Columns
 
-    val query = findItemsBase(q, true, Seq.empty)
-
     // inclusive tags are AND-ed
     val tagSelectsIncl = q.tagsInclude
       .map(tid =>
@@ -404,6 +411,7 @@ object QItem {
       if (batch == Batch.all) Fragment.empty
       else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
 
+    val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
     val frag =
       query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
     logger.trace(s"List $batch items: $frag")
@@ -413,6 +421,7 @@ object QItem {
   case class SelectedItem(itemId: Ident, weight: Double)
   def findSelectedItems(
       q: Query,
+      maxNoteLen: Int,
       items: Set[SelectedItem]
   ): Stream[ConnectionIO, ListItem] =
     if (items.isEmpty) Stream.empty
@@ -425,6 +434,7 @@ object QItem {
       val from = findItemsBase(
         q,
         true,
+        maxNoteLen,
         Seq(fr"tids.weight"),
         ("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
       ) ++

modules/webapp/src/main/elm/App/View.elm

@@ -146,7 +146,7 @@ viewQueue model =
 
 viewUserSettings : Model -> Html Msg
 viewUserSettings model =
-    Html.map UserSettingsMsg (Page.UserSettings.View.view model.uiSettings model.userSettingsModel)
+    Html.map UserSettingsMsg (Page.UserSettings.View.view model.flags model.uiSettings model.userSettingsModel)
 
 
 viewCollectiveSettings : Model -> Html Msg

modules/webapp/src/main/elm/Comp/ItemCardList.elm

@@ -197,6 +197,22 @@ viewItem settings item =
                     )
                 ]
             ]
+        , div
+            [ classList
+                [ ( "content", True )
+                , ( "invisible hidden"
+                  , settings.itemSearchNoteLength
+                        <= 0
+                        || Util.String.isNothingOrBlank item.notes
+                  )
+                ]
+            ]
+            [ span [ class "small-info" ]
+                [ Maybe.withDefault "" item.notes
+                    |> Util.String.ellipsis settings.itemSearchNoteLength
+                    |> text
+                ]
+            ]
         , div [ class "content" ]
             [ div [ class "ui horizontal list" ]
                 [ div

modules/webapp/src/main/elm/Comp/UiSettingsForm.elm

@@ -12,7 +12,7 @@ import Comp.ColorTagger
 import Comp.IntField
 import Data.Color exposing (Color)
 import Data.Flags exposing (Flags)
-import Data.UiSettings exposing (StoredUiSettings, UiSettings)
+import Data.UiSettings exposing (UiSettings)
 import Dict exposing (Dict)
 import Html exposing (..)
 import Html.Attributes exposing (..)
@@ -27,6 +27,8 @@ type alias Model =
     , tagColors : Dict String Color
     , tagColorModel : Comp.ColorTagger.Model
     , nativePdfPreview : Bool
+    , itemSearchNoteLength : Maybe Int
+    , searchNoteLengthModel : Comp.IntField.Model
     }
 
 
@@ -36,7 +38,7 @@ init flags settings =
       , searchPageSizeModel =
             Comp.IntField.init
                 (Just 10)
-                (Just 500)
+                (Just flags.config.maxPageSize)
                 False
                 "Page size"
       , tagColors = settings.tagCategoryColors
@@ -45,6 +47,13 @@ init flags settings =
                 []
                 Data.Color.all
       , nativePdfPreview = settings.nativePdfPreview
+      , itemSearchNoteLength = Just settings.itemSearchNoteLength
+      , searchNoteLengthModel =
+            Comp.IntField.init
+                (Just 0)
+                (Just flags.config.maxNoteLength)
+                False
+                "Max. Note Length"
       }
     , Api.getTags flags "" GetTagsResp
     )
@@ -55,6 +64,7 @@ type Msg
     | TagColorMsg Comp.ColorTagger.Msg
     | GetTagsResp (Result Http.Error TagList)
     | TogglePdfPreview
+    | NoteLengthMsg Comp.IntField.Msg
 
 
 
@@ -80,6 +90,22 @@ update sett msg model =
             in
             ( model_, nextSettings )
 
+        NoteLengthMsg lm ->
+            let
+                ( m, n ) =
+                    Comp.IntField.update lm model.searchNoteLengthModel
+
+                nextSettings =
+                    Maybe.map (\len -> { sett | itemSearchNoteLength = len }) n
+
+                model_ =
+                    { model
+                        | searchNoteLengthModel = m
+                        , itemSearchNoteLength = n
+                    }
+            in
+            ( model_, nextSettings )
+
         TagColorMsg lm ->
             let
                 ( m_, d_ ) =
@@ -139,19 +165,32 @@ tagColorViewOpts =
     }
 
 
-view : UiSettings -> Model -> Html Msg
+view : Flags -> UiSettings -> Model -> Html Msg
-view _ model =
+view flags _ model =
     div [ class "ui form" ]
         [ div [ class "ui dividing header" ]
             [ text "Item Search"
             ]
         , Html.map SearchPageSizeMsg
             (Comp.IntField.viewWithInfo
-                "Maximum results in one page when searching items."
+                ("Maximum results in one page when searching items. At most "
+                    ++ String.fromInt flags.config.maxPageSize
+                    ++ "."
+                )
                 model.itemSearchPageSize
                 "field"
                 model.searchPageSizeModel
             )
+        , Html.map NoteLengthMsg
+            (Comp.IntField.viewWithInfo
+                ("Maximum size of the item notes to display in card view. Between 0 - "
+                    ++ String.fromInt flags.config.maxNoteLength
+                    ++ "."
+                )
+                model.itemSearchNoteLength
+                "field"
+                model.searchNoteLengthModel
+            )
         , div [ class "ui dividing header" ]
             [ text "Item Detail"
             ]

modules/webapp/src/main/elm/Comp/UiSettingsManage.elm

@@ -115,10 +115,10 @@ isSuccess model =
     Maybe.map .success model.message == Just True
 
 
-view : UiSettings -> String -> Model -> Html Msg
+view : Flags -> UiSettings -> String -> Model -> Html Msg
-view settings classes model =
+view flags settings classes model =
     div [ class classes ]
-        [ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view settings model.formModel)
+        [ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view flags settings model.formModel)
         , div [ class "ui divider" ] []
         , button
             [ class "ui primary button"

modules/webapp/src/main/elm/Data/Flags.elm

@@ -16,6 +16,8 @@ type alias Config =
     , docspellAssetPath : String
     , integrationEnabled : Bool
     , fullTextSearchEnabled : Bool
+    , maxPageSize : Int
+    , maxNoteLength : Int
     }
 
 

modules/webapp/src/main/elm/Data/UiSettings.elm

@@ -26,6 +26,7 @@ type alias StoredUiSettings =
     { itemSearchPageSize : Maybe Int
     , tagCategoryColors : List ( String, String )
     , nativePdfPreview : Bool
+    , itemSearchNoteLength : Maybe Int
     }
 
 
@@ -40,6 +41,7 @@ type alias UiSettings =
     { itemSearchPageSize : Int
     , tagCategoryColors : Dict String Color
     , nativePdfPreview : Bool
+    , itemSearchNoteLength : Int
     }
 
 
@@ -48,6 +50,7 @@ defaults =
     { itemSearchPageSize = 60
     , tagCategoryColors = Dict.empty
     , nativePdfPreview = False
+    , itemSearchNoteLength = 0
     }
 
 
@@ -64,6 +67,8 @@ merge given fallback =
             )
             fallback.tagCategoryColors
     , nativePdfPreview = given.nativePdfPreview
+    , itemSearchNoteLength =
+        choose given.itemSearchNoteLength fallback.itemSearchNoteLength
     }
 
 
@@ -79,6 +84,7 @@ toStoredUiSettings settings =
         Dict.map (\_ -> Data.Color.toString) settings.tagCategoryColors
             |> Dict.toList
     , nativePdfPreview = settings.nativePdfPreview
+    , itemSearchNoteLength = Just settings.itemSearchNoteLength
     }
 
 

modules/webapp/src/main/elm/Page/UserSettings/View.elm

@@ -6,6 +6,7 @@ import Comp.ImapSettingsManage
 import Comp.NotificationManage
 import Comp.ScanMailboxManage
 import Comp.UiSettingsManage
+import Data.Flags exposing (Flags)
 import Data.UiSettings exposing (UiSettings)
 import Html exposing (..)
 import Html.Attributes exposing (..)
@@ -14,8 +15,8 @@ import Page.UserSettings.Data exposing (..)
 import Util.Html exposing (classActive)
 
 
-view : UiSettings -> Model -> Html Msg
+view : Flags -> UiSettings -> Model -> Html Msg
-view settings model =
+view flags settings model =
     div [ class "usersetting-page ui padded grid" ]
         [ div [ class "sixteen wide mobile four wide tablet four wide computer column" ]
             [ h4 [ class "ui top attached ablue-comp header" ]
@@ -51,7 +52,7 @@ view settings model =
                         viewScanMailboxManage settings model
 
                     Just UiSettingsTab ->
-                        viewUiSettings settings model
+                        viewUiSettings flags settings model
 
                     Nothing ->
                         []
@@ -72,8 +73,8 @@ makeTab model tab header icon =
         ]
 
 
-viewUiSettings : UiSettings -> Model -> List (Html Msg)
+viewUiSettings : Flags -> UiSettings -> Model -> List (Html Msg)
-viewUiSettings settings model =
+viewUiSettings flags settings model =
     [ h2 [ class "ui header" ]
         [ i [ class "cog icon" ] []
         , text "UI Settings"
@@ -84,6 +85,7 @@ viewUiSettings settings model =
         ]
     , Html.map UiSettingsMsg
         (Comp.UiSettingsManage.view
+            flags
             settings
             "ui segment"
             model.uiSettingsModel

modules/webapp/src/main/elm/Util/String.elm

@@ -1,6 +1,8 @@
 module Util.String exposing
     ( crazyEncode
     , ellipsis
+    , isBlank
+    , isNothingOrBlank
     , underscoreToSpace
     , withDefault
     )
@@ -31,7 +33,7 @@ ellipsis len str =
         str
 
     else
-        String.left (len - 3) str ++ "..."
+        String.left (len - 1) str ++ "…"
 
 
 withDefault : String -> String -> String
@@ -46,3 +48,14 @@ withDefault default str =
 underscoreToSpace : String -> String
 underscoreToSpace str =
     String.replace "_" " " str
+
+
+isBlank : String -> Bool
+isBlank s =
+    s == "" || (String.trim s == "")
+
+
+isNothingOrBlank : Maybe String -> Bool
+isNothingOrBlank ms =
+    Maybe.map isBlank ms
+        |> Maybe.withDefault True

nix/module-server.nix

@@ -14,6 +14,7 @@ let
     app-id = "rest1";
     base-url = "http://localhost:7880";
     max-item-page-size = 200;
+    max-note-length = 180;
     bind = {
       address = "localhost";
       port = 7880;
@@ -124,6 +125,17 @@ in {
         '';
       };
 
+      max-note-length = mkOption {
+        type = types.int;
+        default = defaults.max-note-length;
+        description = ''
+          The number of characters to return for each item notes when
+          searching. Item notes may be very long, when returning them with
+          all the results from a search, they add quite some data to return.
+          In order to keep this low, a limit can be defined here.
+        '';
+      };
+
       bind = mkOption {
         type = types.submodule({
           options = {