Merge pull request #211 from eikek/notes-in-listitem

Notes in listitem
This commit is contained in:
mergify[bot] 2020-08-04 22:25:17 +00:00 committed by GitHub
commit 17e072ef6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 174 additions and 42 deletions

View File

@ -15,20 +15,20 @@ import docspell.store.records.RJob
trait OFulltext[F[_]] {
def findItems(
def findItems(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
fts: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@ -92,7 +92,7 @@ object OFulltext {
else queue.insertIfNew(job) *> joex.notifyAllNodes
} yield ()
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
ftsQ: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@ -120,7 +120,7 @@ object OFulltext {
.transact(
QItem.findItemsWithTags(
account.collective,
QItem.findSelectedItems(QItem.Query.empty(account), select)
QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select)
)
)
.take(batch.limit.toLong)
@ -133,15 +133,23 @@ object OFulltext {
} yield res
}
def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem])
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItems(maxNoteLen),
convertFtsData[ListItem]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map({ case (li, fd) => FtsItem(li, fd) })
.compile
.toVector
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
ftsQ: FtsInput,
batch: Batch
@ -150,7 +158,7 @@ object OFulltext {
q,
ftsQ,
batch.first,
itemSearch.findItemsWithTags,
itemSearch.findItemsWithTags(maxNoteLen),
convertFtsData[ListItemWithTags]
)
.drop(batch.offset.toLong)

View File

@ -17,10 +17,12 @@ import doobie.implicits._
trait OItemSearch[F[_]] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
@ -97,14 +99,16 @@ object OItemSearch {
.transact(QItem.findItem(id))
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, batch).take(batch.limit.toLong))
.transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, batch)
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, maxNoteLen: Int, batch)
store
.transact(
QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)

View File

@ -82,7 +82,7 @@ object NotifyDueItemsTask {
)
res <-
ctx.store
.transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong))
.transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
.compile
.toVector
} yield res

View File

@ -3860,6 +3860,10 @@ components:
type: array
items:
$ref: "#/components/schemas/Tag"
notes:
description: |
Some prefix of the item notes.
type: string
highlighting:
description: |
Optional contextual information of a search query. Each

View File

@ -24,6 +24,12 @@ docspell.server {
# depending on the available resources.
max-item-page-size = 200
# The number of characters to return for each item notes when
# searching. Item notes may be very long, when returning them with
# all the results from a search, they add quite some data to return.
# In order to keep this low, a limit can be defined here.
max-note-length = 180
# Authentication.
auth {

View File

@ -16,6 +16,7 @@ case class Config(
auth: Login.Config,
integrationEndpoint: Config.IntegrationEndpoint,
maxItemPageSize: Int,
maxNoteLength: Int,
fullTextSearch: Config.FullTextSearch
)

View File

@ -197,6 +197,7 @@ trait Conversions {
i.folder.map(mkIdName),
i.fileCount,
Nil,
i.notes,
Nil
)

View File

@ -40,7 +40,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItems(
items <- backend.fulltext.findItems(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@ -49,7 +49,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItems(
items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -67,7 +67,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItemsWithTags(
items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@ -76,7 +76,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItemsWithTags(
items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -92,7 +92,7 @@ object ItemRoutes {
case q if q.length > 1 =>
val ftsIn = OFulltext.FtsInput(q)
for {
items <- backend.fulltext.findIndexOnly(
items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
ftsIn,
user.account,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)

View File

@ -15,7 +15,9 @@ case class Flags(
signupMode: SignupConfig.Mode,
docspellAssetPath: String,
integrationEnabled: Boolean,
fullTextSearchEnabled: Boolean
fullTextSearchEnabled: Boolean,
maxPageSize: Int,
maxNoteLength: Int
)
object Flags {
@ -26,7 +28,9 @@ object Flags {
cfg.backend.signup.mode,
s"/app/assets/docspell-webapp/${BuildInfo.version}",
cfg.integrationEndpoint.enabled,
cfg.fullTextSearch.enabled
cfg.fullTextSearch.enabled,
cfg.maxItemPageSize,
cfg.maxNoteLength
)
implicit val jsonEncoder: Encoder[Flags] =

View File

@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def decrement[A: Put](a: A): Fragment =
f ++ fr"=" ++ f ++ fr"- $a"
def substring(from: Int, many: Int): Fragment =
if (many <= 0 || from < 0) fr"${""}"
else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)"
}

View File

@ -156,7 +156,8 @@ object QItem {
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef],
folder: Option[IdRef]
folder: Option[IdRef],
notes: Option[String]
)
case class Query(
@ -228,6 +229,7 @@ object QItem {
private def findItemsBase(
q: Query,
distinct: Boolean,
noteMaxLen: Int,
moreCols: Seq[Fragment],
ctes: (String, Fragment)*
): Fragment = {
@ -264,6 +266,9 @@ object QItem {
EC.name.prefix("e1").f,
FC.id.prefix("f1").f,
FC.name.prefix("f1").f,
// sql uses 1 for first character
IC.notes.prefix("i").substring(1, noteMaxLen),
// last column is only for sorting
q.orderAsc match {
case Some(co) =>
coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
@ -307,14 +312,16 @@ object QItem {
fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1"))
}
def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = {
def findItems(
q: Query,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns
val PC = RPerson.Columns
val OC = ROrganization.Columns
val EC = REquipment.Columns
val query = findItemsBase(q, true, Seq.empty)
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude
.map(tid =>
@ -404,6 +411,7 @@ object QItem {
if (batch == Batch.all) Fragment.empty
else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
val frag =
query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
logger.trace(s"List $batch items: $frag")
@ -413,6 +421,7 @@ object QItem {
case class SelectedItem(itemId: Ident, weight: Double)
def findSelectedItems(
q: Query,
maxNoteLen: Int,
items: Set[SelectedItem]
): Stream[ConnectionIO, ListItem] =
if (items.isEmpty) Stream.empty
@ -425,6 +434,7 @@ object QItem {
val from = findItemsBase(
q,
true,
maxNoteLen,
Seq(fr"tids.weight"),
("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
) ++

View File

@ -146,7 +146,7 @@ viewQueue model =
viewUserSettings : Model -> Html Msg
viewUserSettings model =
Html.map UserSettingsMsg (Page.UserSettings.View.view model.uiSettings model.userSettingsModel)
Html.map UserSettingsMsg (Page.UserSettings.View.view model.flags model.uiSettings model.userSettingsModel)
viewCollectiveSettings : Model -> Html Msg

View File

@ -197,6 +197,22 @@ viewItem settings item =
)
]
]
, div
[ classList
[ ( "content", True )
, ( "invisible hidden"
, settings.itemSearchNoteLength
<= 0
|| Util.String.isNothingOrBlank item.notes
)
]
]
[ span [ class "small-info" ]
[ Maybe.withDefault "" item.notes
|> Util.String.ellipsis settings.itemSearchNoteLength
|> text
]
]
, div [ class "content" ]
[ div [ class "ui horizontal list" ]
[ div

View File

@ -12,7 +12,7 @@ import Comp.ColorTagger
import Comp.IntField
import Data.Color exposing (Color)
import Data.Flags exposing (Flags)
import Data.UiSettings exposing (StoredUiSettings, UiSettings)
import Data.UiSettings exposing (UiSettings)
import Dict exposing (Dict)
import Html exposing (..)
import Html.Attributes exposing (..)
@ -27,6 +27,8 @@ type alias Model =
, tagColors : Dict String Color
, tagColorModel : Comp.ColorTagger.Model
, nativePdfPreview : Bool
, itemSearchNoteLength : Maybe Int
, searchNoteLengthModel : Comp.IntField.Model
}
@ -36,7 +38,7 @@ init flags settings =
, searchPageSizeModel =
Comp.IntField.init
(Just 10)
(Just 500)
(Just flags.config.maxPageSize)
False
"Page size"
, tagColors = settings.tagCategoryColors
@ -45,6 +47,13 @@ init flags settings =
[]
Data.Color.all
, nativePdfPreview = settings.nativePdfPreview
, itemSearchNoteLength = Just settings.itemSearchNoteLength
, searchNoteLengthModel =
Comp.IntField.init
(Just 0)
(Just flags.config.maxNoteLength)
False
"Max. Note Length"
}
, Api.getTags flags "" GetTagsResp
)
@ -55,6 +64,7 @@ type Msg
| TagColorMsg Comp.ColorTagger.Msg
| GetTagsResp (Result Http.Error TagList)
| TogglePdfPreview
| NoteLengthMsg Comp.IntField.Msg
@ -80,6 +90,22 @@ update sett msg model =
in
( model_, nextSettings )
NoteLengthMsg lm ->
let
( m, n ) =
Comp.IntField.update lm model.searchNoteLengthModel
nextSettings =
Maybe.map (\len -> { sett | itemSearchNoteLength = len }) n
model_ =
{ model
| searchNoteLengthModel = m
, itemSearchNoteLength = n
}
in
( model_, nextSettings )
TagColorMsg lm ->
let
( m_, d_ ) =
@ -139,19 +165,32 @@ tagColorViewOpts =
}
view : UiSettings -> Model -> Html Msg
view _ model =
view : Flags -> UiSettings -> Model -> Html Msg
view flags _ model =
div [ class "ui form" ]
[ div [ class "ui dividing header" ]
[ text "Item Search"
]
, Html.map SearchPageSizeMsg
(Comp.IntField.viewWithInfo
"Maximum results in one page when searching items."
("Maximum results in one page when searching items. At most "
++ String.fromInt flags.config.maxPageSize
++ "."
)
model.itemSearchPageSize
"field"
model.searchPageSizeModel
)
, Html.map NoteLengthMsg
(Comp.IntField.viewWithInfo
("Maximum size of the item notes to display in card view. Between 0 - "
++ String.fromInt flags.config.maxNoteLength
++ "."
)
model.itemSearchNoteLength
"field"
model.searchNoteLengthModel
)
, div [ class "ui dividing header" ]
[ text "Item Detail"
]

View File

@ -115,10 +115,10 @@ isSuccess model =
Maybe.map .success model.message == Just True
view : UiSettings -> String -> Model -> Html Msg
view settings classes model =
view : Flags -> UiSettings -> String -> Model -> Html Msg
view flags settings classes model =
div [ class classes ]
[ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view settings model.formModel)
[ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view flags settings model.formModel)
, div [ class "ui divider" ] []
, button
[ class "ui primary button"

View File

@ -16,6 +16,8 @@ type alias Config =
, docspellAssetPath : String
, integrationEnabled : Bool
, fullTextSearchEnabled : Bool
, maxPageSize : Int
, maxNoteLength : Int
}

View File

@ -26,6 +26,7 @@ type alias StoredUiSettings =
{ itemSearchPageSize : Maybe Int
, tagCategoryColors : List ( String, String )
, nativePdfPreview : Bool
, itemSearchNoteLength : Maybe Int
}
@ -40,6 +41,7 @@ type alias UiSettings =
{ itemSearchPageSize : Int
, tagCategoryColors : Dict String Color
, nativePdfPreview : Bool
, itemSearchNoteLength : Int
}
@ -48,6 +50,7 @@ defaults =
{ itemSearchPageSize = 60
, tagCategoryColors = Dict.empty
, nativePdfPreview = False
, itemSearchNoteLength = 0
}
@ -64,6 +67,8 @@ merge given fallback =
)
fallback.tagCategoryColors
, nativePdfPreview = given.nativePdfPreview
, itemSearchNoteLength =
choose given.itemSearchNoteLength fallback.itemSearchNoteLength
}
@ -79,6 +84,7 @@ toStoredUiSettings settings =
Dict.map (\_ -> Data.Color.toString) settings.tagCategoryColors
|> Dict.toList
, nativePdfPreview = settings.nativePdfPreview
, itemSearchNoteLength = Just settings.itemSearchNoteLength
}

View File

@ -6,6 +6,7 @@ import Comp.ImapSettingsManage
import Comp.NotificationManage
import Comp.ScanMailboxManage
import Comp.UiSettingsManage
import Data.Flags exposing (Flags)
import Data.UiSettings exposing (UiSettings)
import Html exposing (..)
import Html.Attributes exposing (..)
@ -14,8 +15,8 @@ import Page.UserSettings.Data exposing (..)
import Util.Html exposing (classActive)
view : UiSettings -> Model -> Html Msg
view settings model =
view : Flags -> UiSettings -> Model -> Html Msg
view flags settings model =
div [ class "usersetting-page ui padded grid" ]
[ div [ class "sixteen wide mobile four wide tablet four wide computer column" ]
[ h4 [ class "ui top attached ablue-comp header" ]
@ -51,7 +52,7 @@ view settings model =
viewScanMailboxManage settings model
Just UiSettingsTab ->
viewUiSettings settings model
viewUiSettings flags settings model
Nothing ->
[]
@ -72,8 +73,8 @@ makeTab model tab header icon =
]
viewUiSettings : UiSettings -> Model -> List (Html Msg)
viewUiSettings settings model =
viewUiSettings : Flags -> UiSettings -> Model -> List (Html Msg)
viewUiSettings flags settings model =
[ h2 [ class "ui header" ]
[ i [ class "cog icon" ] []
, text "UI Settings"
@ -84,6 +85,7 @@ viewUiSettings settings model =
]
, Html.map UiSettingsMsg
(Comp.UiSettingsManage.view
flags
settings
"ui segment"
model.uiSettingsModel

View File

@ -1,6 +1,8 @@
module Util.String exposing
( crazyEncode
, ellipsis
, isBlank
, isNothingOrBlank
, underscoreToSpace
, withDefault
)
@ -31,7 +33,7 @@ ellipsis len str =
str
else
String.left (len - 3) str ++ "..."
String.left (len - 1) str ++ ""
withDefault : String -> String -> String
@ -46,3 +48,14 @@ withDefault default str =
underscoreToSpace : String -> String
underscoreToSpace str =
String.replace "_" " " str
isBlank : String -> Bool
isBlank s =
s == "" || (String.trim s == "")
isNothingOrBlank : Maybe String -> Bool
isNothingOrBlank ms =
Maybe.map isBlank ms
|> Maybe.withDefault True

View File

@ -14,6 +14,7 @@ let
app-id = "rest1";
base-url = "http://localhost:7880";
max-item-page-size = 200;
max-note-length = 180;
bind = {
address = "localhost";
port = 7880;
@ -124,6 +125,17 @@ in {
'';
};
max-note-length = mkOption {
type = types.int;
default = defaults.max-note-length;
description = ''
The number of characters to return for each item notes when
searching. Item notes may be very long, when returning them with
all the results from a search, they add quite some data to return.
In order to keep this low, a limit can be defined here.
'';
};
bind = mkOption {
type = types.submodule({
options = {