From 3e0914ece7ddbc47dd11a44459229b1bc039a286 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 11 Apr 2021 13:57:49 +0200 Subject: [PATCH] Correctly count tag categories If multiple tags of the same category are applied to the same item, just summing tag counts will produce the wrong results as now items are counted multiple times. --- .../docspell/backend/ops/OCollective.scala | 3 ++ .../src/main/resources/docspell-openapi.yml | 29 ++++++++++++++++- .../restserver/conv/Conversions.scala | 4 +++ .../scala/docspell/store/qb/DBFunction.scala | 4 +-- .../main/scala/docspell/store/qb/DSL.scala | 5 ++- .../store/qb/impl/DBFunctionBuilder.scala | 5 +-- .../store/queries/CategoryCount.scala | 3 ++ .../scala/docspell/store/queries/QItem.scala | 31 ++++++++++++++++++- .../store/queries/SearchSummary.scala | 1 + .../docspell/store/queries/TagCount.scala | 2 +- 10 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index d7417fd9..5579445e 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -71,6 +71,9 @@ object OCollective { type TagCount = docspell.store.queries.TagCount val TagCount = docspell.store.queries.TagCount + type CategoryCount = docspell.store.queries.CategoryCount + val CategoryCount = docspell.store.queries.CategoryCount + type InsightData = QCollective.InsightData val insightData = QCollective.InsightData diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index d579b278..252831ac 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -4242,6 +4242,7 @@ components: required: - count - tagCloud + - tagCategoryCloud - fieldStats - folderStats properties: @@ -4250,6 +4251,8 @@ components: format: int32 tagCloud: $ref: "#/components/schemas/TagCloud" + tagCategoryCloud: + $ref: "#/components/schemas/NameCloud" fieldStats: type: array items: @@ -4354,7 +4357,7 @@ components: $ref: "#/components/schemas/TagCount" TagCount: description: | - Generic structure for counting something. + Structure for counting tags. required: - tag - count @@ -4364,6 +4367,30 @@ components: count: type: integer format: int32 + + NameCloud: + description: | + A set of counters. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/NameCount" + NameCount: + description: | + Generic structure for counting something. + required: + - name + - count + properties: + name: + type: string + count: + type: integer + format: int32 + AttachmentMeta: description: | Extracted meta data of an attachment. diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index ed4d5f0e..99b01138 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -31,6 +31,7 @@ trait Conversions { SearchStats( sum.count, mkTagCloud(sum.tags), + mkTagCategoryCloud(sum.cats), sum.fields.map(mkFieldStats), sum.folders.map(mkFolderStats) ) @@ -63,6 +64,9 @@ trait Conversions { def mkTagCloud(tags: List[OCollective.TagCount]) = TagCloud(tags.map(tc => TagCount(mkTag(tc.tag), tc.count))) + def mkTagCategoryCloud(tags: List[OCollective.CategoryCount]) = + NameCloud(tags.map(tc => NameCount(tc.category, tc.count))) + // attachment meta def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = AttachmentMeta( diff --git a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala index 40db91b2..705bf959 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DBFunction.scala @@ -9,11 +9,11 @@ object DBFunction { val countAll: DBFunction = CountAll def countAs[A](column: Column[A]): DBFunction = - Count(column) + Count(column, false) case object CountAll extends DBFunction - case class Count(column: Column[_]) extends DBFunction + case class Count(column: Column[_], distinct: Boolean) extends DBFunction case class Max(expr: SelectExpr) extends DBFunction diff --git a/modules/store/src/main/scala/docspell/store/qb/DSL.scala b/modules/store/src/main/scala/docspell/store/qb/DSL.scala index d2e78be1..c8a883f4 100644 --- a/modules/store/src/main/scala/docspell/store/qb/DSL.scala +++ b/modules/store/src/main/scala/docspell/store/qb/DSL.scala @@ -63,7 +63,10 @@ trait DSL extends DoobieMeta { FromExpr.From(sel, alias) def count(c: Column[_]): DBFunction = - DBFunction.Count(c) + DBFunction.Count(c, false) + + def countDistinct(c: Column[_]): DBFunction = + DBFunction.Count(c, true) def countAll: DBFunction = DBFunction.CountAll diff --git a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala index c57a9ac3..eba78b27 100644 --- a/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala +++ b/modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala @@ -13,8 +13,9 @@ object DBFunctionBuilder extends CommonBuilder { case DBFunction.CountAll => sql"COUNT(*)" - case DBFunction.Count(col) => - sql"COUNT(" ++ column(col) ++ fr")" + case DBFunction.Count(col, distinct) => + if (distinct) sql"COUNT(DISTINCT " ++ column(col) ++ fr")" + else sql"COUNT(" ++ column(col) ++ fr")" case DBFunction.Max(expr) => sql"MAX(" ++ SelectExprBuilder.build(expr) ++ fr")" diff --git a/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala b/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala new file mode 100644 index 00000000..fd502037 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/CategoryCount.scala @@ -0,0 +1,3 @@ +package docspell.store.queries + +final case class CategoryCount(category: String, count: Int) diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 637a6890..b9335055 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -190,9 +190,38 @@ object QItem { for { count <- searchCountSummary(today)(q) tags <- searchTagSummary(today)(q) + cats <- searchTagCategorySummary(today)(q) fields <- searchFieldSummary(today)(q) folders <- searchFolderSummary(today)(q) - } yield SearchSummary(count, tags, fields, folders) + } yield SearchSummary(count, tags, cats, fields, folders) + + def searchTagCategorySummary( + today: LocalDate + )(q: Query): ConnectionIO[List[CategoryCount]] = { + val tagFrom = + from(ti) + .innerJoin(tag, tag.tid === ti.tagId) + .innerJoin(i, i.id === ti.itemId) + + val tagCloud = + findItemsBase(q.fix, today, 0).unwrap + .withSelect(select(tag.category).append(countDistinct(i.id).as("num"))) + .changeFrom(_.prepend(tagFrom)) + .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) + .groupBy(tag.category) + .build + .query[CategoryCount] + .to[List] + + // the previous query starts from tags, so items with tag-count=0 + // are not included they are fetched separately + for { + existing <- tagCloud + allCats <- RTag.listCategories(q.fix.account.collective) + other = allCats.diff(existing.map(_.category)) + } yield existing ++ other.map(CategoryCount(_, 0)) + + } def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index 0530c211..7beb1724 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -3,6 +3,7 @@ package docspell.store.queries case class SearchSummary( count: Int, tags: List[TagCount], + cats: List[CategoryCount], fields: List[FieldStats], folders: List[FolderCount] ) diff --git a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala index e392f889..b7237f2f 100644 --- a/modules/store/src/main/scala/docspell/store/queries/TagCount.scala +++ b/modules/store/src/main/scala/docspell/store/queries/TagCount.scala @@ -2,4 +2,4 @@ package docspell.store.queries import docspell.store.records.RTag -case class TagCount(tag: RTag, count: Int) +final case class TagCount(tag: RTag, count: Int)