From b50f57f7fe0a68e5a7855f0ef2fc420f5b274df8 Mon Sep 17 00:00:00 2001 From: eikek Date: Thu, 2 Jun 2022 23:10:51 +0200 Subject: [PATCH] Send no fts query if it is disabled --- .../docspell/backend/ops/search/OSearch.scala | 7 ++- .../backend/ops/search/QueryParseResult.scala | 5 +++ .../restserver/routes/ItemRoutes.scala | 3 ++ .../restserver/routes/ItemSearchPart.scala | 41 +++++++++-------- .../scala/docspell/store/fts/TempFtsOps.scala | 45 ++++++++++--------- 5 files changed, 58 insertions(+), 43 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala index dafb2afb..17e7412e 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala @@ -147,7 +147,7 @@ object OSearch { case Some(ftq) => for { timed <- Duration.stopTime[F] - ftq <- createFtsQuery(q.fix.account, batch, ftq) + ftq <- createFtsQuery(q.fix.account, ftq) results <- WeakAsync.liftK[F, ConnectionIO].use { nat => val tempTable = temporaryFtsTable(ftq, nat) @@ -206,7 +206,7 @@ object OSearch { fulltextQuery match { case Some(ftq) => for { - ftq <- createFtsQuery(q.fix.account, Batch.limit(500), ftq) + ftq <- createFtsQuery(q.fix.account, ftq) results <- WeakAsync.liftK[F, ConnectionIO].use { nat => val tempTable = temporaryFtsTable(ftq, nat) store.transact( @@ -221,13 +221,12 @@ object OSearch { private def createFtsQuery( account: AccountId, - batch: Batch, ftq: String ): F[FtsQuery] = store .transact(QFolder.getMemberFolders(account)) .map(folders => - FtsQuery(ftq, account.collective, batch.limit, 0) + FtsQuery(ftq, account.collective, 500, 0) .withFolders(folders) ) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala index 56dc207b..1faf2275 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala @@ -20,6 +20,11 @@ sealed trait QueryParseResult { object QueryParseResult { final case class Success(q: Query, ftq: Option[String]) extends QueryParseResult { + + /** Drop the fulltext search query if disabled. */ + def withFtsEnabled(enabled: Boolean) = + if (enabled || ftq.isEmpty) this else copy(ftq = None) + val get = Some(q -> ftq) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index a312f187..bf182725 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -74,6 +74,7 @@ object ItemRoutes { case req @ POST -> Root / "search" => for { + timed <- Duration.stopTime[F] userQuery <- req.as[ItemQuery] batch = Batch( userQuery.offset.getOrElse(0), @@ -92,6 +93,8 @@ object ItemRoutes { ) fixQuery = Query.Fix(user.account, None, None) resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped) + dur <- timed + _ <- logger.debug(s"Search request: ${dur.formatExact}") } yield resp case req @ POST -> Root / "searchStats" => diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala index c9603b3d..bd386b90 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala @@ -14,8 +14,8 @@ import cats.syntax.all._ import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.backend.ops.search.QueryParseResult -import docspell.common.{SearchMode, Timestamp} -import docspell.query.FulltextExtract +import docspell.common.{Duration, SearchMode, Timestamp} +import docspell.query.FulltextExtract.Result import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.Conversions @@ -44,31 +44,36 @@ final class ItemSearchPart[F[_]: Async]( ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => val userQuery = ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse("")) - - Timestamp - .current[F] - .map(_.toUtcDate) - .flatMap(search(userQuery, _)) + for { + today <- Timestamp.current[F].map(_.toUtcDate) + resp <- search(userQuery, today) + } yield resp case req @ POST -> Root / "search" => for { + timed <- Duration.stopTime[F] userQuery <- req.as[ItemQuery] - today <- Timestamp.current[F] - resp <- search(userQuery, today.toUtcDate) + today <- Timestamp.current[F].map(_.toUtcDate) + resp <- search(userQuery, today) + dur <- timed + _ <- logger.debug(s"Search request: ${dur.formatExact}") } yield resp case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) => val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse("")) - Timestamp - .current[F] - .map(_.toUtcDate) - .flatMap(searchStats(userQuery, _)) + for { + today <- Timestamp.current[F].map(_.toUtcDate) + resp <- searchStats(userQuery, today) + } yield resp case req @ POST -> Root / "searchStats" => for { + timed <- Duration.stopTime[F] userQuery <- req.as[ItemQuery] today <- Timestamp.current[F].map(_.toUtcDate) resp <- searchStats(userQuery, today) + dur <- timed + _ <- logger.debug(s"Search stats request: ${dur.formatExact}") } yield resp } @@ -105,7 +110,7 @@ final class ItemSearchPart[F[_]: Async]( ) // order is always by date unless q is empty and ftq is not - // TODO this is not obvious from the types and an impl detail. + // TODO this should be given explicitly by the result ftsOrder = res.q.cond.isEmpty && res.ftq.isDefined resp <- Ok(convert(items, batch, limitCapped, ftsOrder)) @@ -119,20 +124,18 @@ final class ItemSearchPart[F[_]: Async]( ): Either[F[Response[F]], QueryParseResult.Success] = backend.search.parseQueryString(authToken.account, mode, userQuery.query) match { case s: QueryParseResult.Success => - Right(s) + Right(s.withFtsEnabled(cfg.fullTextSearch.enabled)) case QueryParseResult.ParseFailed(err) => Left(BadRequest(BasicResult(false, s"Invalid query: $err"))) - case QueryParseResult.FulltextMismatch(FulltextExtract.Result.TooMany) => + case QueryParseResult.FulltextMismatch(Result.TooMany) => Left( BadRequest( BasicResult(false, "Only one fulltext search expression is allowed.") ) ) - case QueryParseResult.FulltextMismatch( - FulltextExtract.Result.UnsupportedPosition - ) => + case QueryParseResult.FulltextMismatch(Result.UnsupportedPosition) => Left( BadRequest( BasicResult( diff --git a/modules/store/src/main/scala/docspell/store/fts/TempFtsOps.scala b/modules/store/src/main/scala/docspell/store/fts/TempFtsOps.scala index b32f57ba..e18d7fb4 100644 --- a/modules/store/src/main/scala/docspell/store/fts/TempFtsOps.scala +++ b/modules/store/src/main/scala/docspell/store/fts/TempFtsOps.scala @@ -6,8 +6,8 @@ package docspell.store.fts -import cats.Foldable import cats.syntax.all._ +import cats.{Foldable, Monad} import fs2.{Pipe, Stream} import docspell.common.Duration @@ -38,10 +38,10 @@ private[fts] object TempFtsOps { timed <- Stream.eval(Duration.stopTime[ConnectionIO]) tt <- Stream.eval(createTable(db, name)) n <- in.through(tt.insert).foldMonoid - _ <- Stream.eval(tt.createIndex) + _ <- if (n > 500) Stream.eval(tt.createIndex) else Stream(()) duration <- Stream.eval(timed) _ <- Stream.eval( - logger.info( + logger.debug( s"Creating temporary fts table ($n elements) took: ${duration.formatExact}" ) ) @@ -122,25 +122,30 @@ private[fts] object TempFtsOps { "(?,?,?)" :: res } .mkString(",") - val sql = - s"""INSERT INTO ${table.tableName} - | (${table.id.name}, ${table.score.name}, ${table.context.name}) - | VALUES $values""".stripMargin + if (values.isEmpty) Monad[ConnectionIO].pure(0) + else { + val sql = + s"""INSERT INTO ${table.tableName} + | (${table.id.name}, ${table.score.name}, ${table.context.name}) + | VALUES $values""".stripMargin - val encoder = io.circe.Encoder[ContextEntry] - doobie.free.FC.raw { conn => - val pst = conn.prepareStatement(sql) - rows.foldl(0) { (index, row) => - pst.setString(index + 1, row.id.id) - row.score - .map(d => pst.setDouble(index + 2, d)) - .getOrElse(pst.setNull(index + 2, java.sql.Types.DOUBLE)) - row.context - .map(c => pst.setString(index + 3, encoder(c).noSpaces)) - .getOrElse(pst.setNull(index + 3, java.sql.Types.VARCHAR)) - index + 3 + val encoder = io.circe.Encoder[ContextEntry] + doobie.free.FC.raw { conn => + val pst = conn.prepareStatement(sql) + rows.foldl(0) { (index, row) => + pst.setString(index + 1, row.id.id) + row.score + .fold(pst.setNull(index + 2, java.sql.Types.DOUBLE))(d => + pst.setDouble(index + 2, d) + ) + row.context + .fold(pst.setNull(index + 3, java.sql.Types.VARCHAR))(c => + pst.setString(index + 3, encoder(c).noSpaces) + ) + index + 3 + } + pst.executeUpdate() } - pst.executeUpdate() } }