From 3e0914ece7ddbc47dd11a44459229b1bc039a286 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 11 Apr 2021 13:57:49 +0200 Subject: [PATCH 1/3] Correctly count tag categories If multiple tags of the same category are applied to the same item, just summing tag counts will produce the wrong results as now items are counted multiple times. --- .../docspell/backend/ops/OCollective.scala | 3 ++ .../src/main/resources/docspell-openapi.yml | 29 ++++++++++++++++- .../restserver/conv/Conversions.scala | 4 +++ .../scala/docspell/store/qb/DBFunction.scala | 4 +-- .../main/scala/docspell/store/qb/DSL.scala | 5 ++- .../store/qb/impl/DBFunctionBuilder.scala | 5 +-- .../store/queries/CategoryCount.scala | 3 ++ .../scala/docspell/store/queries/QItem.scala | 31 ++++++++++++++++++- .../store/queries/SearchSummary.scala | 1 + .../docspell/store/queries/TagCount.scala | 2 +- 10 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index d7417fd9..5579445e 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -71,6 +71,9 @@ object OCollective { type TagCount = docspell.store.queries.TagCount val TagCount = docspell.store.queries.TagCount + type CategoryCount = docspell.store.queries.CategoryCount + val CategoryCount = docspell.store.queries.CategoryCount + type InsightData = QCollective.InsightData val insightData = QCollective.InsightData diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index d579b278..252831ac 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -4242,6 +4242,7 @@ components: required: - count - tagCloud + - tagCategoryCloud - fieldStats - folderStats properties: @@ -4250,6 +4251,8 @@ components: format: int32 tagCloud: $ref: "#/components/schemas/TagCloud" + tagCategoryCloud: + $ref: "#/components/schemas/NameCloud" fieldStats: type: array items: @@ -4354,7 +4357,7 @@ components: $ref: "#/components/schemas/TagCount" TagCount: description: | - Generic structure for counting something. + Structure for counting tags. required: - tag - count @@ -4364,6 +4367,30 @@ components: count: type: integer format: int32 + + NameCloud: + description: | + A set of counters. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/NameCount" + NameCount: + description: | + Generic structure for counting something. + required: + - name + - count + properties: + name: + type: string + count: + type: integer + format: int32 + AttachmentMeta: description: | Extracted meta data of an attachment. diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index ed4d5f0e..99b01138 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -31,6 +31,7 @@ trait Conversions { SearchStats( sum.count, mkTagCloud(sum.tags), + mkTagCategoryCloud(sum.cats), sum.fields.map(mkFieldStats), sum.folders.map(mkFolderStats) ) @@ -63,6 +64,9 @@ trait Conversions { def mkTagCloud(tags: List[OCollective.TagCount]) = TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count))) + def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) = + NameCloud(tags.map(tc => NameCount(tc.category, tc.count))) + // attachment meta def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = AttachmentMeta( diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 40db91b2..705bf959 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -9,11 +9,11 @@ object DBFunction { val countAll: DBFunction = CountAll def countAs[A](column: Column[A]): DBFunction = - Count(column) + Count(column, false) case object CountAll extends DBFunction - case class Count(column: Column[_]) extends DBFunction + case class Count(column: Column[_], distinct: Boolean) extends DBFunction case class Max(expr: SelectExpr) extends DBFunction diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index d2e78be1..c8a883f4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -63,7 +63,10 @@ trait DSL extends DoobieMeta { FromExpr.From(sel, alias) def count(c: Column[_]): DBFunction = - DBFunction.Count(c) + DBFunction.Count(c, false) + + def countDistinct(c: Column[_]): DBFunction = + DBFunction.Count(c, true) def countAll: DBFunction = DBFunction.CountAll diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index c57a9ac3..eba78b27 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -13,8 +13,9 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.CountAll => sql"COUNT(*)" - case DBFunction.Count(col) => - sql"COUNT(" ++ column(col) ++ fr")" + case DBFunction.Count(col, distinct) => + if (distinct) sql"COUNT(DISTINCT " ++ column(col) ++ fr")" + else sql"COUNT(" ++ column(col) ++ fr")" case DBFunction.Max(expr) => sql"MAX(" ++ SelectExprBuilder.build(expr) ++ fr")" diff --git a/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala b/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala new file mode 100644 index 00000000..fd502037 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala @@ -0,0 +1,3 @@ +package docspell.store.queries + +final case class CategoryCount(category: String, count: Int) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 637a6890..b9335055 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -190,9 +190,38 @@ object QItem { for { count <- searchCountSummary(today)(q) tags <- searchTagSummary(today)(q) + cats <- searchTagCategorySummary(today)(q) fields <- searchFieldSummary(today)(q) folders <- searchFolderSummary(today)(q) - } yield SearchSummary(count, tags, fields, folders) + } yield SearchSummary(count, tags, cats, fields, folders) + + def searchTagCategorySummary( + today: LocalDate + )(q: Query): ConnectionIO[List[CategoryCount]] = { + val tagFrom = + from(ti) + .innerJoin(tag, tag.tid === ti.tagId) + .innerJoin(i, i.id === ti.itemId) + + val tagCloud = + findItemsBase(q.fix, today, 0).unwrap + .withSelect(select(tag.category).append(countDistinct(i.id).as("num"))) + .changeFrom(_.prepend(tagFrom)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) + .groupBy(tag.category) + .build + .query[CategoryCount] + .to[List] + + // the previous query starts from tags, so items with tag-count=0 + // are not included they are fetched separately + for { + existing <- tagCloud + allCats <- RTag.listCategories(q.fix.account.collective) + other = allCats.diff(existing.map(_.category)) + } yield existing ++ other.map(CategoryCount(_, 0)) + + } def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index 0530c211..7beb1724 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -3,6 +3,7 @@ package docspell.store.queries case class SearchSummary( count: Int, tags: List[TagCount], + cats: List[CategoryCount], fields: List[FieldStats], folders: List[FolderCount] ) diff --git a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala index e392f889..b7237f2f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala +++ b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala @@ -2,4 +2,4 @@ package docspell.store.queries import docspell.store.records.RTag -case class TagCount(tag: RTag, count: Int) +final case class TagCount(tag: RTag, count: Int) From 39ed246a42953175fd6e656070b06c7c2fb97a49 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 11 Apr 2021 14:19:07 +0200 Subject: [PATCH 2/3] Use correct category count in search menu --- .../webapp/src/main/elm/Comp/SearchMenu.elm | 15 +++- .../webapp/src/main/elm/Comp/TagSelect.elm | 79 ++++++------------- 2 files changed, 35 insertions(+), 59 deletions(-) diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index b0355286..768374f0 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -92,7 +92,7 @@ type TextSearchModel init : Flags -> Model init flags = - { tagSelectModel = Comp.TagSelect.init [] [] + { tagSelectModel = Comp.TagSelect.init [] [] [] [] , tagSelection = Comp.TagSelect.emptySelection , directionModel = Comp.Dropdown.makeSingleList @@ -483,7 +483,9 @@ updateDrop ddm flags settings msg model = GetAllTagsResp (Ok stats) -> let tagSel = - Comp.TagSelect.modifyAll stats.tagCloud.items model.tagSelectModel + Comp.TagSelect.modifyAll stats.tagCloud.items + stats.tagCategoryCloud.items + model.tagSelectModel in { model = { model | tagSelectModel = tagSel } , cmd = Cmd.none @@ -500,9 +502,14 @@ updateDrop ddm flags settings msg model = GetStatsResp (Ok stats) -> let - selectModel = + tagCount = List.sortBy .count stats.tagCloud.items - |> Comp.TagSelect.modifyCount model.tagSelectModel + + catCount = + List.sortBy .count stats.tagCategoryCloud.items + + selectModel = + Comp.TagSelect.modifyCount model.tagSelectModel tagCount catCount model_ = { model diff --git a/modules/webapp/src/main/elm/Comp/TagSelect.elm b/modules/webapp/src/main/elm/Comp/TagSelect.elm index 79bd149a..31fa8cda 100644 --- a/modules/webapp/src/main/elm/Comp/TagSelect.elm +++ b/modules/webapp/src/main/elm/Comp/TagSelect.elm @@ -1,6 +1,5 @@ module Comp.TagSelect exposing - ( Category - , Model + ( Model , Msg , Selection , WorkModel @@ -18,6 +17,7 @@ module Comp.TagSelect exposing , viewTagsDrop2 ) +import Api.Model.NameCount exposing (NameCount) import Api.Model.Tag exposing (Tag) import Api.Model.TagCount exposing (TagCount) import Data.Icons as I @@ -38,9 +38,9 @@ import Util.Maybe type alias Model = { availableTags : Dict String TagCount - , availableCats : Dict String Category + , availableCats : Dict String NameCount , tagCounts : List TagCount - , categoryCounts : List Category + , categoryCounts : List NameCount , filterTerm : Maybe String , expandedTags : Bool , expandedCats : Bool @@ -48,23 +48,16 @@ type alias Model = } -type alias Category = - { name : String - , count : Int - } - - -init : List TagCount -> List TagCount -> Model -init allTags tags = +init : List TagCount -> List NameCount -> List TagCount -> List NameCount -> Model +init allTags allCats tags cats = { availableTags = List.map (\e -> ( e.tag.id, e )) allTags |> Dict.fromList - , availableCats = sumCategories allTags + , availableCats = + List.map (\e -> ( e.name, e )) allCats + |> Dict.fromList , tagCounts = tags - , categoryCounts = - sumCategories tags - |> Dict.toList - |> List.map Tuple.second + , categoryCounts = cats , filterTerm = Nothing , expandedTags = False , expandedCats = False @@ -72,24 +65,23 @@ init allTags tags = } -modifyAll : List TagCount -> Model -> Model -modifyAll allTags model = +modifyAll : List TagCount -> List NameCount -> Model -> Model +modifyAll allTags allCats model = { model | availableTags = List.map (\e -> ( e.tag.id, e )) allTags |> Dict.fromList - , availableCats = sumCategories allTags + , availableCats = + List.map (\e -> ( e.name, e )) allCats + |> Dict.fromList } -modifyCount : Model -> List TagCount -> Model -modifyCount model tags = +modifyCount : Model -> List TagCount -> List NameCount -> Model +modifyCount model tags cats = { model | tagCounts = tags - , categoryCounts = - sumCategories tags - |> Dict.toList - |> List.map Tuple.second + , categoryCounts = cats } @@ -108,34 +100,11 @@ toggleTag id = ToggleTag id -sumCategories : List TagCount -> Dict String Category -sumCategories tags = - let - filterCat tc = - Maybe.map (\cat -> Category cat tc.count) tc.tag.category - - withCats = - List.filterMap filterCat tags - - sum cat mc = - Maybe.map ((+) cat.count) mc - |> Maybe.withDefault cat.count - |> Just - - sumCounts cat dict = - Dict.update cat.name (sum cat) dict - - cats = - List.foldl sumCounts Dict.empty withCats - in - Dict.map (\name -> \count -> Category name count) cats - - type alias Selection = { includeTags : List TagCount , excludeTags : List TagCount - , includeCats : List Category - , excludeCats : List Category + , includeCats : List NameCount + , excludeCats : List NameCount } @@ -145,7 +114,7 @@ emptySelection = type alias WorkModel = - { filteredCats : List Category + { filteredCats : List NameCount , filteredTags : List TagCount , selectedTags : Dict String Bool , selectedCats : Dict String Bool @@ -166,7 +135,7 @@ orderTagCountStable model tagCounts = List.sortBy order tagCounts -orderCatCountStable : Model -> List Category -> List Category +orderCatCountStable : Model -> List NameCount -> List NameCount orderCatCountStable model catCounts = let order cat = @@ -193,7 +162,7 @@ removeEmptyTagCounts sel tagCounts = List.filter (\tc -> isSelected tc || tc.count > 0) tagCounts -removeEmptyCatCounts : Selection -> List Category -> List Category +removeEmptyCatCounts : Selection -> List NameCount -> List NameCount removeEmptyCatCounts sel catCounts = let selected = @@ -548,7 +517,7 @@ viewTagItem2 ddm settings model tag = ] -viewCategoryItem2 : UiSettings -> WorkModel -> Category -> Html Msg +viewCategoryItem2 : UiSettings -> WorkModel -> NameCount -> Html Msg viewCategoryItem2 settings model cat = let state = From f40fa0419cad84d3ebf14b143d27b4874a0c8e75 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 11 Apr 2021 14:40:18 +0200 Subject: [PATCH 3/3] Assign dependency prs for notification --- .mergify.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index a2ea0870..341a451d 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -1,4 +1,12 @@ pull_request_rules: + - name: assign and label scala-steward's PRs + conditions: + - author=scala-steward + actions: + assign: + users: [eikek] + label: + add: ["type: dependencies"] - name: automatically merge Scala Steward PRs on CI success conditions: - author=scala-steward