TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-11-03 18:00:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #211 from eikek/notes-in-listitem

Browse Source 
Notes in listitem
This commit is contained in:
mergify[bot]
2020-08-04 22:25:17 +00:00
committed by GitHub
parent f1e776ae3d 08f953dd52
commit 17e072ef6e
20 changed files with 174 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
View File

@@ -15,20 +15,20 @@ import docspell.store.records.RJob
trait OFulltext[F[_]] {
def findItems(
def findItems(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
fts: OFulltext.FtsInput,
batch: Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
fts: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@@ -92,7 +92,7 @@ object OFulltext {
else queue.insertIfNew(job) *> joex.notifyAllNodes
} yield ()
def findIndexOnly(
def findIndexOnly(maxNoteLen: Int)(
ftsQ: OFulltext.FtsInput,
account: AccountId,
batch: Batch
@@ -120,7 +120,7 @@ object OFulltext {
.transact(
QItem.findItemsWithTags(
account.collective,
QItem.findSelectedItems(QItem.Query.empty(account), select)
QItem.findSelectedItems(QItem.Query.empty(account), maxNoteLen, select)
)
)
.take(batch.limit.toLong)
@@ -133,15 +133,23 @@ object OFulltext {
} yield res
}
def findItems(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(q, ftsQ, batch.first, itemSearch.findItems, convertFtsData[ListItem])
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: Batch): F[Vector[FtsItem]] =
findItemsFts(
q,
ftsQ,
batch.first,
itemSearch.findItems(maxNoteLen),
convertFtsData[ListItem]
)
.drop(batch.offset.toLong)
.take(batch.limit.toLong)
.map({ case (li, fd) => FtsItem(li, fd) })
.compile
.toVector
def findItemsWithTags(
def findItemsWithTags(maxNoteLen: Int)(
q: Query,
ftsQ: FtsInput,
batch: Batch
@@ -150,7 +158,7 @@ object OFulltext {
q,
ftsQ,
batch.first,
itemSearch.findItemsWithTags,
itemSearch.findItemsWithTags(maxNoteLen),
convertFtsData[ListItemWithTags]
)
.drop(batch.offset.toLong)

16
modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala
View File

@@ -17,10 +17,12 @@ import doobie.implicits._
trait OItemSearch[F[_]] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
@@ -97,14 +99,16 @@ object OItemSearch {
.transact(QItem.findItem(id))
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
def findItems(maxNoteLen: Int)(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, batch).take(batch.limit.toLong))
.transact(QItem.findItems(q, maxNoteLen, batch).take(batch.limit.toLong))
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, batch)
def findItemsWithTags(
maxNoteLen: Int
)(q: Query, batch: Batch): F[Vector[ListItemWithTags]] = {
val search = QItem.findItems(q, maxNoteLen: Int, batch)
store
.transact(
QItem.findItemsWithTags(q.account.collective, search).take(batch.limit.toLong)

2
modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
View File

@@ -82,7 +82,7 @@ object NotifyDueItemsTask {
)
res <-
ctx.store
.transact(QItem.findItems(q, Batch.limit(maxItems)).take(maxItems.toLong))
.transact(QItem.findItems(q, 0, Batch.limit(maxItems)).take(maxItems.toLong))
.compile
.toVector
} yield res

4
modules/restapi/src/main/resources/docspell-openapi.yml
View File

@@ -3860,6 +3860,10 @@ components:
type: array
items:
$ref: "#/components/schemas/Tag"
notes:
description: |
Some prefix of the item notes.
type: string
highlighting:
description: |
Optional contextual information of a search query. Each

6
modules/restserver/src/main/resources/reference.conf
View File

@@ -24,6 +24,12 @@ docspell.server {
# depending on the available resources.
max-item-page-size = 200
# The number of characters to return for each item notes when
# searching. Item notes may be very long, when returning them with
# all the results from a search, they add quite some data to return.
# In order to keep this low, a limit can be defined here.
max-note-length = 180
# Authentication.
auth {

1
modules/restserver/src/main/scala/docspell/restserver/Config.scala
View File

@@ -16,6 +16,7 @@ case class Config(
auth: Login.Config,
integrationEndpoint: Config.IntegrationEndpoint,
maxItemPageSize: Int,
maxNoteLength: Int,
fullTextSearch: Config.FullTextSearch
)

1
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@@ -197,6 +197,7 @@ trait Conversions {
i.folder.map(mkIdName),
i.fileCount,
Nil,
i.notes,
Nil
)

10
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@@ -40,7 +40,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItems(
items <- backend.fulltext.findItems(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -49,7 +49,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItems(
items <- backend.itemSearch.findItems(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@@ -67,7 +67,7 @@ object ItemRoutes {
resp <- mask.fullText match {
case Some(fq) if cfg.fullTextSearch.enabled =>
for {
items <- backend.fulltext.findItemsWithTags(
items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)(
query,
OFulltext.FtsInput(fq),
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
@@ -76,7 +76,7 @@ object ItemRoutes {
} yield ok
case _ =>
for {
items <- backend.itemSearch.findItemsWithTags(
items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@@ -92,7 +92,7 @@ object ItemRoutes {
case q if q.length > 1 =>
val ftsIn = OFulltext.FtsInput(q)
for {
items <- backend.fulltext.findIndexOnly(
items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)(
ftsIn,
user.account,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)

8
modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala
View File

@@ -15,7 +15,9 @@ case class Flags(
signupMode: SignupConfig.Mode,
docspellAssetPath: String,
integrationEnabled: Boolean,
fullTextSearchEnabled: Boolean
fullTextSearchEnabled: Boolean,
maxPageSize: Int,
maxNoteLength: Int
)
object Flags {
@@ -26,7 +28,9 @@ object Flags {
cfg.backend.signup.mode,
s"/app/assets/docspell-webapp/${BuildInfo.version}",
cfg.integrationEndpoint.enabled,
cfg.fullTextSearch.enabled
cfg.fullTextSearch.enabled,
cfg.maxItemPageSize,
cfg.maxNoteLength
)
implicit val jsonEncoder: Encoder[Flags] =

4
modules/store/src/main/scala/docspell/store/impl/Column.scala
View File

@@ -121,4 +121,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def decrement[A: Put](a: A): Fragment =
f ++ fr"=" ++ f ++ fr"- $a"
def substring(from: Int, many: Int): Fragment =
if (many <= 0 || from < 0) fr"${""}"
else fr"SUBSTRING(" ++ f ++ fr"FROM $from FOR $many)"
}

18
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@@ -156,7 +156,8 @@ object QItem {
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef],
folder: Option[IdRef]
folder: Option[IdRef],
notes: Option[String]
)
case class Query(
@@ -228,6 +229,7 @@ object QItem {
private def findItemsBase(
q: Query,
distinct: Boolean,
noteMaxLen: Int,
moreCols: Seq[Fragment],
ctes: (String, Fragment)*
): Fragment = {
@@ -264,6 +266,9 @@ object QItem {
EC.name.prefix("e1").f,
FC.id.prefix("f1").f,
FC.name.prefix("f1").f,
// sql uses 1 for first character
IC.notes.prefix("i").substring(1, noteMaxLen),
// last column is only for sorting
q.orderAsc match {
case Some(co) =>
coalesce(co(IC).prefix("i").f, IC.created.prefix("i").f)
@@ -307,14 +312,16 @@ object QItem {
fr"LEFT JOIN folders f1 ON" ++ IC.folder.prefix("i").is(FC.id.prefix("f1"))
}
def findItems(q: Query, batch: Batch): Stream[ConnectionIO, ListItem] = {
def findItems(
q: Query,
maxNoteLen: Int,
batch: Batch
): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns
val PC = RPerson.Columns
val OC = ROrganization.Columns
val EC = REquipment.Columns
val query = findItemsBase(q, true, Seq.empty)
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude
.map(tid =>
@@ -404,6 +411,7 @@ object QItem {
if (batch == Batch.all) Fragment.empty
else fr"LIMIT ${batch.limit} OFFSET ${batch.offset}"
val query = findItemsBase(q, true, maxNoteLen, Seq.empty)
val frag =
query ++ fr"WHERE" ++ cond ++ order ++ limitOffset
logger.trace(s"List $batch items: $frag")
@@ -413,6 +421,7 @@ object QItem {
case class SelectedItem(itemId: Ident, weight: Double)
def findSelectedItems(
q: Query,
maxNoteLen: Int,
items: Set[SelectedItem]
): Stream[ConnectionIO, ListItem] =
if (items.isEmpty) Stream.empty
@@ -425,6 +434,7 @@ object QItem {
val from = findItemsBase(
q,
true,
maxNoteLen,
Seq(fr"tids.weight"),
("tids(item_id, weight)", fr"(VALUES" ++ values ++ fr")")
) ++

2
modules/webapp/src/main/elm/App/View.elm
View File

@@ -146,7 +146,7 @@ viewQueue model =
viewUserSettings : Model -> Html Msg
viewUserSettings model =
Html.map UserSettingsMsg (Page.UserSettings.View.view model.uiSettings model.userSettingsModel)
Html.map UserSettingsMsg (Page.UserSettings.View.view model.flags model.uiSettings model.userSettingsModel)
viewCollectiveSettings : Model -> Html Msg

16
modules/webapp/src/main/elm/Comp/ItemCardList.elm
View File

@@ -197,6 +197,22 @@ viewItem settings item =
)
]
]
, div
[ classList
[ ( "content", True )
, ( "invisible hidden"
, settings.itemSearchNoteLength
<= 0
|| Util.String.isNothingOrBlank item.notes
)
]
]
[ span [ class "small-info" ]
[ Maybe.withDefault "" item.notes
|> Util.String.ellipsis settings.itemSearchNoteLength
|> text
]
]
, div [ class "content" ]
[ div [ class "ui horizontal list" ]
[ div

49
modules/webapp/src/main/elm/Comp/UiSettingsForm.elm
View File

@@ -12,7 +12,7 @@ import Comp.ColorTagger
import Comp.IntField
import Data.Color exposing (Color)
import Data.Flags exposing (Flags)
import Data.UiSettings exposing (StoredUiSettings, UiSettings)
import Data.UiSettings exposing (UiSettings)
import Dict exposing (Dict)
import Html exposing (..)
import Html.Attributes exposing (..)
@@ -27,6 +27,8 @@ type alias Model =
, tagColors : Dict String Color
, tagColorModel : Comp.ColorTagger.Model
, nativePdfPreview : Bool
, itemSearchNoteLength : Maybe Int
, searchNoteLengthModel : Comp.IntField.Model
}
@@ -36,7 +38,7 @@ init flags settings =
, searchPageSizeModel =
Comp.IntField.init
(Just 10)
(Just 500)
(Just flags.config.maxPageSize)
False
"Page size"
, tagColors = settings.tagCategoryColors
@@ -45,6 +47,13 @@ init flags settings =
[]
Data.Color.all
, nativePdfPreview = settings.nativePdfPreview
, itemSearchNoteLength = Just settings.itemSearchNoteLength
, searchNoteLengthModel =
Comp.IntField.init
(Just 0)
(Just flags.config.maxNoteLength)
False
"Max. Note Length"
}
, Api.getTags flags "" GetTagsResp
)
@@ -55,6 +64,7 @@ type Msg
| TagColorMsg Comp.ColorTagger.Msg
| GetTagsResp (Result Http.Error TagList)
| TogglePdfPreview
| NoteLengthMsg Comp.IntField.Msg
@@ -80,6 +90,22 @@ update sett msg model =
in
( model_, nextSettings )
NoteLengthMsg lm ->
let
( m, n ) =
Comp.IntField.update lm model.searchNoteLengthModel
nextSettings =
Maybe.map (\len -> { sett | itemSearchNoteLength = len }) n
model_ =
{ model
| searchNoteLengthModel = m
, itemSearchNoteLength = n
}
in
( model_, nextSettings )
TagColorMsg lm ->
let
( m_, d_ ) =
@@ -139,19 +165,32 @@ tagColorViewOpts =
}
view : UiSettings -> Model -> Html Msg
view _ model =
view : Flags -> UiSettings -> Model -> Html Msg
view flags _ model =
div [ class "ui form" ]
[ div [ class "ui dividing header" ]
[ text "Item Search"
]
, Html.map SearchPageSizeMsg
(Comp.IntField.viewWithInfo
"Maximum results in one page when searching items."
("Maximum results in one page when searching items. At most "
++ String.fromInt flags.config.maxPageSize
++ "."
)
model.itemSearchPageSize
"field"
model.searchPageSizeModel
)
, Html.map NoteLengthMsg
(Comp.IntField.viewWithInfo
("Maximum size of the item notes to display in card view. Between 0 - "
++ String.fromInt flags.config.maxNoteLength
++ "."
)
model.itemSearchNoteLength
"field"
model.searchNoteLengthModel
)
, div [ class "ui dividing header" ]
[ text "Item Detail"
]

6
modules/webapp/src/main/elm/Comp/UiSettingsManage.elm
View File

@@ -115,10 +115,10 @@ isSuccess model =
Maybe.map .success model.message == Just True
view : UiSettings -> String -> Model -> Html Msg
view settings classes model =
view : Flags -> UiSettings -> String -> Model -> Html Msg
view flags settings classes model =
div [ class classes ]
[ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view settings model.formModel)
[ Html.map UiSettingsFormMsg (Comp.UiSettingsForm.view flags settings model.formModel)
, div [ class "ui divider" ] []
, button
[ class "ui primary button"

2
modules/webapp/src/main/elm/Data/Flags.elm
View File

@@ -16,6 +16,8 @@ type alias Config =
, docspellAssetPath : String
, integrationEnabled : Bool
, fullTextSearchEnabled : Bool
, maxPageSize : Int
, maxNoteLength : Int
}

6
modules/webapp/src/main/elm/Data/UiSettings.elm
View File

@@ -26,6 +26,7 @@ type alias StoredUiSettings =
{ itemSearchPageSize : Maybe Int
, tagCategoryColors : List ( String, String )
, nativePdfPreview : Bool
, itemSearchNoteLength : Maybe Int
}
@@ -40,6 +41,7 @@ type alias UiSettings =
{ itemSearchPageSize : Int
, tagCategoryColors : Dict String Color
, nativePdfPreview : Bool
, itemSearchNoteLength : Int
}
@@ -48,6 +50,7 @@ defaults =
{ itemSearchPageSize = 60
, tagCategoryColors = Dict.empty
, nativePdfPreview = False
, itemSearchNoteLength = 0
}
@@ -64,6 +67,8 @@ merge given fallback =
)
fallback.tagCategoryColors
, nativePdfPreview = given.nativePdfPreview
, itemSearchNoteLength =
choose given.itemSearchNoteLength fallback.itemSearchNoteLength
}
@@ -79,6 +84,7 @@ toStoredUiSettings settings =
Dict.map (\_ -> Data.Color.toString) settings.tagCategoryColors
|> Dict.toList
, nativePdfPreview = settings.nativePdfPreview
, itemSearchNoteLength = Just settings.itemSearchNoteLength
}

12
modules/webapp/src/main/elm/Page/UserSettings/View.elm
View File

@@ -6,6 +6,7 @@ import Comp.ImapSettingsManage
import Comp.NotificationManage
import Comp.ScanMailboxManage
import Comp.UiSettingsManage
import Data.Flags exposing (Flags)
import Data.UiSettings exposing (UiSettings)
import Html exposing (..)
import Html.Attributes exposing (..)
@@ -14,8 +15,8 @@ import Page.UserSettings.Data exposing (..)
import Util.Html exposing (classActive)
view : UiSettings -> Model -> Html Msg
view settings model =
view : Flags -> UiSettings -> Model -> Html Msg
view flags settings model =
div [ class "usersetting-page ui padded grid" ]
[ div [ class "sixteen wide mobile four wide tablet four wide computer column" ]
[ h4 [ class "ui top attached ablue-comp header" ]
@@ -51,7 +52,7 @@ view settings model =
viewScanMailboxManage settings model
Just UiSettingsTab ->
viewUiSettings settings model
viewUiSettings flags settings model
Nothing ->
[]
@@ -72,8 +73,8 @@ makeTab model tab header icon =
]
viewUiSettings : UiSettings -> Model -> List (Html Msg)
viewUiSettings settings model =
viewUiSettings : Flags -> UiSettings -> Model -> List (Html Msg)
viewUiSettings flags settings model =
[ h2 [ class "ui header" ]
[ i [ class "cog icon" ] []
, text "UI Settings"
@@ -84,6 +85,7 @@ viewUiSettings settings model =
]
, Html.map UiSettingsMsg
(Comp.UiSettingsManage.view
flags
settings
"ui segment"
model.uiSettingsModel

15
modules/webapp/src/main/elm/Util/String.elm
View File

@@ -1,6 +1,8 @@
module Util.String exposing
( crazyEncode
, ellipsis
, isBlank
, isNothingOrBlank
, underscoreToSpace
, withDefault
)
@@ -31,7 +33,7 @@ ellipsis len str =
str
else
String.left (len - 3) str ++ "..."
String.left (len - 1) str ++ "…"
withDefault : String -> String -> String
@@ -46,3 +48,14 @@ withDefault default str =
underscoreToSpace : String -> String
underscoreToSpace str =
String.replace "_" " " str
isBlank : String -> Bool
isBlank s =
s == "" || (String.trim s == "")
isNothingOrBlank : Maybe String -> Bool
isNothingOrBlank ms =
Maybe.map isBlank ms
|> Maybe.withDefault True

12
nix/module-server.nix
View File

@@ -14,6 +14,7 @@ let
app-id = "rest1";
base-url = "http://localhost:7880";
max-item-page-size = 200;
max-note-length = 180;
bind = {
address = "localhost";
port = 7880;
@@ -124,6 +125,17 @@ in {
'';
};
max-note-length = mkOption {
type = types.int;
default = defaults.max-note-length;
description = ''
The number of characters to return for each item notes when
searching. Item notes may be very long, when returning them with
all the results from a search, they add quite some data to return.
In order to keep this low, a limit can be defined here.
'';
};
bind = mkOption {
type = types.submodule({
options = {