Add summary for fulltext searches

This commit is contained in:
Eike Kettner 2020-12-17 00:11:33 +01:00
parent 8d7b3c7d74
commit 6346bf6a34
4 changed files with 100 additions and 20 deletions
build.sbt
modules
backend/src/main/scala/docspell/backend/ops
restserver/src/main/scala/docspell/restserver/routes
webapp/src/main/elm/Page/Home

@ -445,7 +445,8 @@ val joex = project
buildInfoPackage := "docspell.joex",
reStart / javaOptions ++= Seq(
s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}"
)
),
Revolver.enableDebugging(port = 5051, suspend = false)
)
.dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr)
@ -492,7 +493,8 @@ val restserver = project
),
reStart / javaOptions ++= Seq(
s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}"
)
),
Revolver.enableDebugging(port = 5050, suspend = false)
)
.dependsOn(restapi, joexapi, backend, webapp, ftssolr)

@ -7,12 +7,15 @@ import fs2.Stream
import docspell.backend.JobFactory
import docspell.backend.ops.OItemSearch._
import docspell.common._
import docspell.common.syntax.all._
import docspell.ftsclient._
import docspell.store.queries.{QFolder, QItem, SelectedItem}
import docspell.store.queue.JobQueue
import docspell.store.records.RJob
import docspell.store.{Store, qb}
import org.log4s.getLogger
trait OFulltext[F[_]] {
def findItems(maxNoteLen: Int)(
@ -34,6 +37,9 @@ trait OFulltext[F[_]] {
batch: qb.Batch
): F[Vector[OFulltext.FtsItemWithTags]]
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
/** Clears the full-text index completely and launches a task that
* indexes all data.
*/
@ -46,6 +52,7 @@ trait OFulltext[F[_]] {
}
object OFulltext {
private[this] val logger = getLogger
case class FtsInput(
query: String,
@ -77,12 +84,14 @@ object OFulltext {
Resource.pure[F, OFulltext[F]](new OFulltext[F] {
def reindexAll: F[Unit] =
for {
_ <- logger.finfo(s"Re-index all.")
job <- JobFactory.reIndexAll[F]
_ <- queue.insertIfNew(job) *> joex.notifyAllNodes
} yield ()
def reindexCollective(account: AccountId): F[Unit] =
for {
_ <- logger.fdebug(s"Re-index collective: $account")
exist <- store.transact(
RJob.findNonFinalByTracker(DocspellSystem.migrationTaskTracker)
)
@ -107,6 +116,7 @@ object OFulltext {
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
)
for {
_ <- logger.ftrace(s"Find index only: ${ftsQ.query}/${batch}")
folders <- store.transact(QFolder.getMemberFolders(account))
ftsR <- fts.search(fq.withFolders(folders))
ftsItems = ftsR.results.groupBy(_.itemId)
@ -133,6 +143,32 @@ object OFulltext {
} yield res
}
def findIndexOnlySummary(
account: AccountId,
ftsQ: OFulltext.FtsInput
): F[SearchSummary] = {
val fq = FtsQuery(
ftsQ.query,
account.collective,
Set.empty,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
for {
folder <- store.transact(QFolder.getMemberFolders(account))
itemIds <- fts
.searchAll(fq.withFolders(folder))
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
q = Query.empty(account).copy(itemIds = itemIds.some)
res <- store.transact(QItem.searchStats(q))
} yield res
}
def findItems(
maxNoteLen: Int
)(q: Query, ftsQ: FtsInput, batch: qb.Batch): F[Vector[FtsItem]] =
@ -167,6 +203,27 @@ object OFulltext {
.compile
.toVector
def findItemsSummary(q: Query, ftsQ: OFulltext.FtsInput): F[SearchSummary] =
for {
search <- itemSearch.findItems(0)(q, Batch.all)
fq = FtsQuery(
ftsQ.query,
q.account.collective,
search.map(_.id).toSet,
Set.empty,
500,
0,
FtsQuery.HighlightSetting.default
)
items <- fts
.searchAll(fq)
.flatMap(r => Stream.emits(r.results.map(_.itemId)))
.compile
.to(Set)
qnext = q.copy(itemIds = items.some)
res <- store.transact(QItem.searchStats(qnext))
} yield res
// Helper
private def findItemsFts[A: ItemId, B](

@ -147,8 +147,19 @@ object ItemRoutes {
for {
mask <- req.as[ItemSearch]
query = Conversions.mkQuery(mask, user.account)
stats <- backend.itemSearch.findItemsSummary(query)
resp <- Ok(Conversions.mkSearchStats(stats))
stats <- mask match {
case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled =>
logger.finfo(s"Make index only summary: $ftq") *>
backend.fulltext.findIndexOnlySummary(
user.account,
OFulltext.FtsInput(ftq.query)
)
case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled =>
backend.fulltext.findItemsSummary(query, OFulltext.FtsInput(fq))
case _ =>
backend.itemSearch.findItemsSummary(query)
}
resp <- Ok(Conversions.mkSearchStats(stats))
} yield resp
case GET -> Root / Ident(id) =>

@ -201,27 +201,37 @@ viewStats _ model =
fields =
List.filter isNumField stats.fieldStats
in
if List.isEmpty fields then
[]
else
[ div [ class "ui container" ]
[ table [ class "ui very basic tiny six column table" ]
[ thead []
[ tr [ class "center aligned" ]
[ th [] []
, th [] [ text "Count" ]
, th [] [ text "Sum" ]
, th [] [ text "Avg" ]
, th [] [ text "Min" ]
, th [] [ text "Max" ]
[ div [ class "ui container" ]
[ div [ class "ui middle aligned grid" ]
[ div [ class "three wide center aligned column" ]
[ div [ class "ui small statistic" ]
[ div [ class "value" ]
[ String.fromInt stats.count |> text
]
, div [ class "label" ]
[ text "Results"
]
]
, tbody []
(List.map statValues fields)
]
, div [ class "thirteen wide column" ]
[ table [ class "ui very basic tiny six column table" ]
[ thead []
[ tr [ class "center aligned" ]
[ th [] []
, th [] [ text "Count" ]
, th [] [ text "Sum" ]
, th [] [ text "Avg" ]
, th [] [ text "Min" ]
, th [] [ text "Max" ]
]
]
, tbody []
(List.map statValues fields)
]
]
]
]
]
viewLeftMenu : Flags -> UiSettings -> Model -> List (Html Msg)