From 6346bf6a344a3dddce7cb41ed56d71cdf240a9d8 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 17 Dec 2020 00:11:33 +0100 Subject: [PATCH] Add summary for fulltext searches --- build.sbt | 6 +- .../docspell/backend/ops/OFulltext.scala | 57 +++++++++++++++++++ .../restserver/routes/ItemRoutes.scala | 15 ++++- .../webapp/src/main/elm/Page/Home/View.elm | 42 ++++++++------ 4 files changed, 100 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index 02f2401b..ff0f60c5 100644 --- a/build.sbt +++ b/build.sbt @@ -445,7 +445,8 @@ val joex = project buildInfoPackage := "docspell.joex", reStart / javaOptions ++= Seq( s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" - ) + ), + Revolver.enableDebugging(port = 5051, suspend = false) ) .dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) @@ -492,7 +493,8 @@ val restserver = project ), reStart / javaOptions ++= Seq( s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" - ) + ), + Revolver.enableDebugging(port = 5050, suspend = false) ) .dependsOn(restapi, joexapi, backend, webapp, ftssolr) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index d7f416d2..52e23571 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -7,12 +7,15 @@ import fs2.Stream import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ +import docspell.common.syntax.all._ import docspell.ftsclient._ import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob import docspell.store.{Store, qb} +import org.log4s.getLogger + trait OFulltext[F[_]] { def findItems(maxNoteLen: Int)( @@ -34,6 +37,9 @@ trait OFulltext[F[_]] { batch: qb.Batch ): F[Vector[OFulltext.FtsItemWithTags]] + def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary] + def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary] + /** Clears the full-text index completely and launches a task that * indexes all data. */ @@ -46,6 +52,7 @@ trait OFulltext[F[_]] { } object OFulltext { + private[this] val logger = getLogger case class FtsInput( query: String, @@ -77,12 +84,14 @@ object OFulltext { Resource.pure[F, OFulltext[F]](new OFulltext[F] { def reindexAll: F[Unit] = for { + _ <- logger.finfo(s"Re-index all.") job <- JobFactory.reIndexAll[F] _ <- queue.insertIfNew(job) *> joex.notifyAllNodes } yield () def reindexCollective(account: AccountId): F[Unit] = for { + _ <- logger.fdebug(s"Re-index collective: $account") exist <- store.transact( RJob.findNonFinalByTracker(DocspellSystem.migrationTaskTracker) ) @@ -107,6 +116,7 @@ object OFulltext { FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost) ) for { + _ <- logger.ftrace(s"Find index only: ${ftsQ.query}/${batch}") folders <- store.transact(QFolder.getMemberFolders(account)) ftsR <- fts.search(fq.withFolders(folders)) ftsItems = ftsR.results.groupBy(_.itemId) @@ -133,6 +143,32 @@ object OFulltext { } yield res } + def findIndexOnlySummary( + account: AccountId, + ftsQ: OFulltext.FtsInput + ): F[SearchSummary] = { + val fq = FtsQuery( + ftsQ.query, + account.collective, + Set.empty, + Set.empty, + 500, + 0, + FtsQuery.HighlightSetting.default + ) + + for { + folder <- store.transact(QFolder.getMemberFolders(account)) + itemIds <- fts + .searchAll(fq.withFolders(folder)) + .flatMap(r => Stream.emits(r.results.map(_.itemId))) + .compile + .to(Set) + q = Query.empty(account).copy(itemIds = itemIds.some) + res <- store.transact(QItem.searchStats(q)) + } yield res + } + def findItems( maxNoteLen: Int )(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] = @@ -167,6 +203,27 @@ object OFulltext { .compile .toVector + def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] = + for { + search <- itemSearch.findItems(0)(q, Batch.all) + fq = FtsQuery( + ftsQ.query, + q.account.collective, + search.map(_.id).toSet, + Set.empty, + 500, + 0, + FtsQuery.HighlightSetting.default + ) + items <- fts + .searchAll(fq) + .flatMap(r => Stream.emits(r.results.map(_.itemId))) + .compile + .to(Set) + qnext = q.copy(itemIds = items.some) + res <- store.transact(QItem.searchStats(qnext)) + } yield res + // Helper private def findItemsFts[A: ItemId, B]( diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index eaa5bb7a..e0b0c5b8 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -147,8 +147,19 @@ object ItemRoutes { for { mask <- req.as[ItemSearch] query = Conversions.mkQuery(mask, user.account) - stats <- backend.itemSearch.findItemsSummary(query) - resp <- Ok(Conversions.mkSearchStats(stats)) + stats <- mask match { + case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => + logger.finfo(s"Make index only summary: $ftq") *> + backend.fulltext.findIndexOnlySummary( + user.account, + OFulltext.FtsInput(ftq.query) + ) + case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => + backend.fulltext.findItemsSummary(query, OFulltext.FtsInput(fq)) + case _ => + backend.itemSearch.findItemsSummary(query) + } + resp <- Ok(Conversions.mkSearchStats(stats)) } yield resp case GET -> Root / Ident(id) => diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index 0f46c3bf..b4c93634 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -201,27 +201,37 @@ viewStats _ model = fields = List.filter isNumField stats.fieldStats in - if List.isEmpty fields then - [] - - else - [ div [ class "ui container" ] - [ table [ class "ui very basic tiny six column table" ] - [ thead [] - [ tr [ class "center aligned" ] - [ th [] [] - , th [] [ text "Count" ] - , th [] [ text "Sum" ] - , th [] [ text "Avg" ] - , th [] [ text "Min" ] - , th [] [ text "Max" ] + [ div [ class "ui container" ] + [ div [ class "ui middle aligned grid" ] + [ div [ class "three wide center aligned column" ] + [ div [ class "ui small statistic" ] + [ div [ class "value" ] + [ String.fromInt stats.count |> text + ] + , div [ class "label" ] + [ text "Results" ] ] - , tbody [] - (List.map statValues fields) + ] + , div [ class "thirteen wide column" ] + [ table [ class "ui very basic tiny six column table" ] + [ thead [] + [ tr [ class "center aligned" ] + [ th [] [] + , th [] [ text "Count" ] + , th [] [ text "Sum" ] + , th [] [ text "Avg" ] + , th [] [ text "Min" ] + , th [] [ text "Max" ] + ] + ] + , tbody [] + (List.map statValues fields) + ] ] ] ] + ] viewLeftMenu : Flags -> UiSettings -> Model -> List (Html Msg)