From 77627534bcac1ab725074d300ca0228ac50b35e3 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 15 Dec 2020 23:11:04 +0100 Subject: [PATCH] Improve on basic search summary --- .../docspell/backend/ops/OItemSearch.scala | 5 +- .../docspell/common/CustomFieldType.scala | 4 +- .../docspell/store/queries/FieldStats.scala | 17 +++++ .../scala/docspell/store/queries/QItem.scala | 70 ++++++++++++++++++- .../store/queries/SearchSummary.scala | 2 +- 5 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/queries/FieldStats.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala index fbfa69d2..9061b87a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala @@ -145,10 +145,7 @@ object OItemSearch { } def findItemsSummary(q: Query): F[SearchSummary] = - for { - tags <- store.transact(QItem.searchTagSummary(q)) - count <- store.transact(QItem.searchCountSummary(q)) - } yield SearchSummary(count, tags) + store.transact(QItem.searchStats(q)) def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = store diff --git a/modules/common/src/main/scala/docspell/common/CustomFieldType.scala b/modules/common/src/main/scala/docspell/common/CustomFieldType.scala index 6c217550..55f9b6bf 100644 --- a/modules/common/src/main/scala/docspell/common/CustomFieldType.scala +++ b/modules/common/src/main/scala/docspell/common/CustomFieldType.scala @@ -2,6 +2,7 @@ package docspell.common import java.time.LocalDate +import cats.data.NonEmptyList import cats.implicits._ import io.circe._ @@ -92,7 +93,8 @@ object CustomFieldType { def bool: CustomFieldType = Bool def money: CustomFieldType = Money - val all: List[CustomFieldType] = List(Text, Numeric, Date, Bool, Money) + val all: NonEmptyList[CustomFieldType] = + NonEmptyList.of(Text, Numeric, Date, Bool, Money) def fromString(str: String): Either[String, CustomFieldType] = str.toLowerCase match { diff --git a/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala b/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala new file mode 100644 index 00000000..4fb90c50 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/FieldStats.scala @@ -0,0 +1,17 @@ +package docspell.store.queries + +import docspell.store.records.RCustomField + +case class FieldStats( + field: RCustomField, + count: Int, + avg: BigDecimal, + sum: BigDecimal, + max: BigDecimal, + min: BigDecimal +) + +object FieldStats { + def apply(field: RCustomField, count: Int): FieldStats = + FieldStats(field, count, BigDecimal(0), BigDecimal(0), BigDecimal(0), BigDecimal(0)) +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index fab67704..e5dbe4b9 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -15,7 +15,7 @@ import docspell.store.records._ import doobie.implicits._ import doobie.{Query => _, _} -import org.log4s._ +import org.log4s.getLogger object QItem { private[this] val logger = getLogger @@ -234,13 +234,20 @@ object QItem { sql.query[ListItem].stream } + def searchStats(q: Query): ConnectionIO[SearchSummary] = + for { + count <- searchCountSummary(q) + tags <- searchTagSummary(q) + fields <- searchFieldSummary(q) + } yield SearchSummary(count, tags, fields) + def searchTagSummary(q: Query): ConnectionIO[List[TagCount]] = { val tagFrom = from(ti) .innerJoin(tag, tag.tid === ti.tagId) .innerJoin(i, i.id === ti.itemId) - findItemsBase(q, 0) + findItemsBase(q, 0).unwrap .withSelect(select(tag.all).append(count(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) .changeWhere(c => c && queryCondition(q)) @@ -251,13 +258,70 @@ object QItem { } def searchCountSummary(q: Query): ConnectionIO[Int] = - findItemsBase(q, 0) + findItemsBase(q, 0).unwrap .withSelect(Nel.of(count(i.id).as("num"))) .changeWhere(c => c && queryCondition(q)) .build .query[Int] .unique + def searchFieldSummary(q: Query): ConnectionIO[List[FieldStats]] = { + val fieldJoin = + from(cv) + .innerJoin(cf, cf.id === cv.field) + .innerJoin(i, i.id === cv.itemId) + + val base = + findItemsBase(q, 0).unwrap + .changeFrom(_.prepend(fieldJoin)) + .changeWhere(c => c && queryCondition(q)) + .groupBy(GroupBy(cf.all)) + + val basicFields = Nel.of( + count(i.id).as("fc"), + lit(0).as("favg"), + lit(0).as("fsum"), + lit(0).as("fmax"), + lit(0).as("fmin") + ) + val valueNum = cast(cv.value.s, "decimal").s + val numericFields = Nel.of( + count(i.id).as("fc"), + avg(valueNum).as("favg"), + sum(valueNum).as("fsum"), + max(valueNum).as("fmax"), + min(valueNum).as("fmin") + ) + + val numTypes = Nel.of(CustomFieldType.money, CustomFieldType.numeric) + val query = + union( + base + .withSelect(select(cf.all).concatNel(basicFields)) + .changeWhere(c => c && cf.ftype.notIn(numTypes)), + base + .withSelect(select(cf.all).concatNel(numericFields)) + .changeWhere(c => c && cf.ftype.in(numTypes)) + ).build.query[FieldStats].to[List] + + val fallback = base + .withSelect(select(cf.all).concatNel(basicFields)) + .build + .query[FieldStats] + .to[List] + + query.attemptSql.flatMap { + case Right(res) => res.pure[ConnectionIO] + case Left(ex) => + Logger + .log4s[ConnectionIO](logger) + .error(ex)( + s"Calculating custom field summary failed. You may have invalid custom field values according to their type." + ) *> + fallback + } + } + def findSelectedItems( q: Query, maxNoteLen: Int, diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index 606ec6ff..36cd2a71 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -1,3 +1,3 @@ package docspell.store.queries -case class SearchSummary(count: Int, tags: List[TagCount]) +case class SearchSummary(count: Int, tags: List[TagCount], fields: List[FieldStats])