Merge pull request #755 from eikek/fix-category-count

Fix category count
2025-04-05 19:09:32 +00:00 · 2021-04-11 14:59:49 +02:00 · 2021-04-11 14:59:49 +02:00 · cdbd65388d
commit cdbd65388d
parent 132730b19c f40fa0419c
13 changed files with 122 additions and 67 deletions
--- a/.mergify.yml
+++ b/.mergify.yml
@ -1,4 +1,12 @@
 pull_request_rules:
+  - name: assign and label scala-steward's PRs
+    conditions:
+      - author=scala-steward
+    actions:
+      assign:
+        users: [eikek]
+      label:
+        add: ["type: dependencies"]
  - name: automatically merge Scala Steward PRs on CI success
    conditions:
      - author=scala-steward
--- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala
@ -71,6 +71,9 @@ object OCollective {
  type TagCount = docspell.store.queries.TagCount
  val TagCount = docspell.store.queries.TagCount

+  type CategoryCount = docspell.store.queries.CategoryCount
+  val CategoryCount = docspell.store.queries.CategoryCount
+
  type InsightData = QCollective.InsightData
  val insightData = QCollective.InsightData

--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@ -4242,6 +4242,7 @@ components:
      required:
        - count
        - tagCloud
+        - tagCategoryCloud
        - fieldStats
        - folderStats
      properties:
@ -4250,6 +4251,8 @@ components:
          format: int32
        tagCloud:
          $ref: "#/components/schemas/TagCloud"
+        tagCategoryCloud:
+          $ref: "#/components/schemas/NameCloud"
        fieldStats:
          type: array
          items:
@ -4354,7 +4357,7 @@ components:
            $ref: "#/components/schemas/TagCount"
    TagCount:
      description: |
-        Generic structure for counting something.
+        Structure for counting tags.
      required:
        - tag
        - count
@ -4364,6 +4367,30 @@ components:
        count:
          type: integer
          format: int32
+
+    NameCloud:
+      description: |
+        A set of counters.
+      required:
+        - items
+      properties:
+        items:
+          type: array
+          items:
+            $ref: "#/components/schemas/NameCount"
+    NameCount:
+      description: |
+        Generic structure for counting something.
+      required:
+        - name
+        - count
+      properties:
+        name:
+          type: string
+        count:
+          type: integer
+          format: int32
+
    AttachmentMeta:
      description: |
        Extracted meta data of an attachment.
--- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@ -31,6 +31,7 @@ trait Conversions {
    SearchStats(
      sum.count,
      mkTagCloud(sum.tags),
+      mkTagCategoryCloud(sum.cats),
      sum.fields.map(mkFieldStats),
      sum.folders.map(mkFolderStats)
    )
@ -63,6 +64,9 @@ trait Conversions {
  def mkTagCloud(tags: List[OCollective.TagCount]) =
    TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count)))

+  def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) =
+    NameCloud(tags.map(tc => NameCount(tc.category, tc.count)))
+
  // attachment meta
  def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta =
    AttachmentMeta(
--- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala
+++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala
@ -9,11 +9,11 @@ object DBFunction {
  val countAll: DBFunction = CountAll

  def countAs[A](column: Column[A]): DBFunction =
-    Count(column)
+    Count(column, false)

  case object CountAll extends DBFunction

-  case class Count(column: Column[_]) extends DBFunction
+  case class Count(column: Column[_], distinct: Boolean) extends DBFunction

  case class Max(expr: SelectExpr) extends DBFunction

--- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala
+++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala
@ -63,7 +63,10 @@ trait DSL extends DoobieMeta {
    FromExpr.From(sel, alias)

  def count(c: Column[_]): DBFunction =
-    DBFunction.Count(c)
+    DBFunction.Count(c, false)
+
+  def countDistinct(c: Column[_]): DBFunction =
+    DBFunction.Count(c, true)

  def countAll: DBFunction =
    DBFunction.CountAll
--- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala
+++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala
@ -13,8 +13,9 @@ object DBFunctionBuilder extends CommonBuilder {
      case DBFunction.CountAll =>
        sql"COUNT(*)"

-      case DBFunction.Count(col) =>
-        sql"COUNT(" ++ column(col) ++ fr")"
+      case DBFunction.Count(col, distinct) =>
+        if (distinct) sql"COUNT(DISTINCT " ++ column(col) ++ fr")"
+        else sql"COUNT(" ++ column(col) ++ fr")"

      case DBFunction.Max(expr) =>
        sql"MAX(" ++ SelectExprBuilder.build(expr) ++ fr")"
--- a/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala
@ -0,0 +1,3 @@
+package docspell.store.queries
+
+final case class CategoryCount(category: String, count: Int)
--- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala
@ -190,9 +190,38 @@ object QItem {
    for {
      count   <- searchCountSummary(today)(q)
      tags    <- searchTagSummary(today)(q)
+      cats    <- searchTagCategorySummary(today)(q)
      fields  <- searchFieldSummary(today)(q)
      folders <- searchFolderSummary(today)(q)
-    } yield SearchSummary(count, tags, fields, folders)
+    } yield SearchSummary(count, tags, cats, fields, folders)
+
+  def searchTagCategorySummary(
+      today: LocalDate
+  )(q: Query): ConnectionIO[List[CategoryCount]] = {
+    val tagFrom =
+      from(ti)
+        .innerJoin(tag, tag.tid === ti.tagId)
+        .innerJoin(i, i.id === ti.itemId)
+
+    val tagCloud =
+      findItemsBase(q.fix, today, 0).unwrap
+        .withSelect(select(tag.category).append(countDistinct(i.id).as("num")))
+        .changeFrom(_.prepend(tagFrom))
+        .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
+        .groupBy(tag.category)
+        .build
+        .query[CategoryCount]
+        .to[List]
+
+    // the previous query starts from tags, so items with tag-count=0
+    // are not included they are fetched separately
+    for {
+      existing <- tagCloud
+      allCats  <- RTag.listCategories(q.fix.account.collective)
+      other = allCats.diff(existing.map(_.category))
+    } yield existing ++ other.map(CategoryCount(_, 0))
+
+  }

  def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = {
    val tagFrom =
--- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala
@ -3,6 +3,7 @@ package docspell.store.queries
 case class SearchSummary(
    count: Int,
    tags: List[TagCount],
+    cats: List[CategoryCount],
    fields: List[FieldStats],
    folders: List[FolderCount]
 )
--- a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala
@ -2,4 +2,4 @@ package docspell.store.queries

 import docspell.store.records.RTag

-case class TagCount(tag: RTag, count: Int)
+final case class TagCount(tag: RTag, count: Int)
--- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm
+++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm
@ -92,7 +92,7 @@ type TextSearchModel

 init : Flags -> Model
 init flags =
-    { tagSelectModel = Comp.TagSelect.init [] []
+    { tagSelectModel = Comp.TagSelect.init [] [] [] []
    , tagSelection = Comp.TagSelect.emptySelection
    , directionModel =
        Comp.Dropdown.makeSingleList
@ -483,7 +483,9 @@ updateDrop ddm flags settings msg model =
        GetAllTagsResp (Ok stats) ->
            let
                tagSel =
-                    Comp.TagSelect.modifyAll stats.tagCloud.items model.tagSelectModel
+                    Comp.TagSelect.modifyAll stats.tagCloud.items
+                        stats.tagCategoryCloud.items
+                        model.tagSelectModel
            in
            { model = { model | tagSelectModel = tagSel }
            , cmd = Cmd.none
@ -500,9 +502,14 @@ updateDrop ddm flags settings msg model =

        GetStatsResp (Ok stats) ->
            let
-                selectModel =
+                tagCount =
                    List.sortBy .count stats.tagCloud.items
-                        |> Comp.TagSelect.modifyCount model.tagSelectModel
+
+                catCount =
+                    List.sortBy .count stats.tagCategoryCloud.items
+
+                selectModel =
+                    Comp.TagSelect.modifyCount model.tagSelectModel tagCount catCount

                model_ =
                    { model
--- a/modules/webapp/src/main/elm/Comp/TagSelect.elm
+++ b/modules/webapp/src/main/elm/Comp/TagSelect.elm
@ -1,6 +1,5 @@
 module Comp.TagSelect exposing
-    ( Category
-    , Model
+    ( Model
    , Msg
    , Selection
    , WorkModel
@ -18,6 +17,7 @@ module Comp.TagSelect exposing
    , viewTagsDrop2
    )

+import Api.Model.NameCount exposing (NameCount)
 import Api.Model.Tag exposing (Tag)
 import Api.Model.TagCount exposing (TagCount)
 import Data.Icons as I
@ -38,9 +38,9 @@ import Util.Maybe

 type alias Model =
    { availableTags : Dict String TagCount
-    , availableCats : Dict String Category
+    , availableCats : Dict String NameCount
    , tagCounts : List TagCount
-    , categoryCounts : List Category
+    , categoryCounts : List NameCount
    , filterTerm : Maybe String
    , expandedTags : Bool
    , expandedCats : Bool
@ -48,23 +48,16 @@ type alias Model =
    }


-type alias Category =
-    { name : String
-    , count : Int
-    }
-
-
-init : List TagCount -> List TagCount -> Model
-init allTags tags =
+init : List TagCount -> List NameCount -> List TagCount -> List NameCount -> Model
+init allTags allCats tags cats =
    { availableTags =
        List.map (\e -> ( e.tag.id, e )) allTags
            |> Dict.fromList
-    , availableCats = sumCategories allTags
+    , availableCats =
+        List.map (\e -> ( e.name, e )) allCats
+            |> Dict.fromList
    , tagCounts = tags
-    , categoryCounts =
-        sumCategories tags
-            |> Dict.toList
-            |> List.map Tuple.second
+    , categoryCounts = cats
    , filterTerm = Nothing
    , expandedTags = False
    , expandedCats = False
@ -72,24 +65,23 @@ init allTags tags =
    }


-modifyAll : List TagCount -> Model -> Model
-modifyAll allTags model =
+modifyAll : List TagCount -> List NameCount -> Model -> Model
+modifyAll allTags allCats model =
    { model
        | availableTags =
            List.map (\e -> ( e.tag.id, e )) allTags
                |> Dict.fromList
-        , availableCats = sumCategories allTags
+        , availableCats =
+            List.map (\e -> ( e.name, e )) allCats
+                |> Dict.fromList
    }


-modifyCount : Model -> List TagCount -> Model
-modifyCount model tags =
+modifyCount : Model -> List TagCount -> List NameCount -> Model
+modifyCount model tags cats =
    { model
        | tagCounts = tags
-        , categoryCounts =
-            sumCategories tags
-                |> Dict.toList
-                |> List.map Tuple.second
+        , categoryCounts = cats
    }


@ -108,34 +100,11 @@ toggleTag id =
    ToggleTag id


-sumCategories : List TagCount -> Dict String Category
-sumCategories tags =
-    let
-        filterCat tc =
-            Maybe.map (\cat -> Category cat tc.count) tc.tag.category
-
-        withCats =
-            List.filterMap filterCat tags
-
-        sum cat mc =
-            Maybe.map ((+) cat.count) mc
-                |> Maybe.withDefault cat.count
-                |> Just
-
-        sumCounts cat dict =
-            Dict.update cat.name (sum cat) dict
-
-        cats =
-            List.foldl sumCounts Dict.empty withCats
-    in
-    Dict.map (\name -> \count -> Category name count) cats
-
-
 type alias Selection =
    { includeTags : List TagCount
    , excludeTags : List TagCount
-    , includeCats : List Category
-    , excludeCats : List Category
+    , includeCats : List NameCount
+    , excludeCats : List NameCount
    }


@ -145,7 +114,7 @@ emptySelection =


 type alias WorkModel =
-    { filteredCats : List Category
+    { filteredCats : List NameCount
    , filteredTags : List TagCount
    , selectedTags : Dict String Bool
    , selectedCats : Dict String Bool
@ -166,7 +135,7 @@ orderTagCountStable model tagCounts =
    List.sortBy order tagCounts


-orderCatCountStable : Model -> List Category -> List Category
+orderCatCountStable : Model -> List NameCount -> List NameCount
 orderCatCountStable model catCounts =
    let
        order cat =
@ -193,7 +162,7 @@ removeEmptyTagCounts sel tagCounts =
    List.filter (\tc -> isSelected tc || tc.count > 0) tagCounts


-removeEmptyCatCounts : Selection -> List Category -> List Category
+removeEmptyCatCounts : Selection -> List NameCount -> List NameCount
 removeEmptyCatCounts sel catCounts =
    let
        selected =
@ -548,7 +517,7 @@ viewTagItem2 ddm settings model tag =
        ]


-viewCategoryItem2 : UiSettings -> WorkModel -> Category -> Html Msg
+viewCategoryItem2 : UiSettings -> WorkModel -> NameCount -> Html Msg
 viewCategoryItem2 settings model cat =
    let
        state =