Prepare for new search logic with feature toggle

build.sbt

@@ -471,6 +471,17 @@ val addonlib = project
   )
   .dependsOn(common, files, loggingScribe)
 
+val ftsclient = project
+  .in(file("modules/fts-client"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .withTestSettings
+  .settings(
+    name := "docspell-fts-client",
+    libraryDependencies ++= Seq.empty
+  )
+  .dependsOn(common, loggingScribe)
+
 val store = project
   .in(file("modules/store"))
   .disablePlugins(RevolverPlugin)
@@ -500,6 +511,7 @@ val store = project
     files,
     notificationApi,
     jsonminiq,
+    ftsclient,
     loggingScribe
   )
 
@@ -623,17 +635,6 @@ val analysis = project
   )
   .dependsOn(common, files % "test->test", loggingScribe)
 
-val ftsclient = project
-  .in(file("modules/fts-client"))
-  .disablePlugins(RevolverPlugin)
-  .settings(sharedSettings)
-  .withTestSettings
-  .settings(
-    name := "docspell-fts-client",
-    libraryDependencies ++= Seq.empty
-  )
-  .dependsOn(common, loggingScribe)
-
 val ftssolr = project
   .in(file("modules/fts-solr"))
   .disablePlugins(RevolverPlugin)

modules/backend/src/main/scala/docspell/backend/BackendApp.scala

@@ -12,6 +12,7 @@ import docspell.backend.BackendCommands.EventContext
 import docspell.backend.auth.Login
 import docspell.backend.fulltext.CreateIndex
 import docspell.backend.ops._
+import docspell.backend.ops.search.OSearch
 import docspell.backend.signup.OSignup
 import docspell.common.bc.BackendCommandRunner
 import docspell.ftsclient.FtsClient
@@ -58,6 +59,7 @@ trait BackendApp[F[_]] {
   def itemLink: OItemLink[F]
   def downloadAll: ODownloadAll[F]
   def addons: OAddons[F]
+  def search: OSearch[F]
 
   def commands(eventContext: Option[EventContext]): BackendCommandRunner[F, Unit]
 }
@@ -130,6 +132,7 @@ object BackendApp {
           joexImpl
         )
       )
+      searchImpl <- Resource.pure(OSearch(store, ftsClient))
     } yield new BackendApp[F] {
       val pubSub = pubSubT
       val login = loginImpl
@@ -162,6 +165,7 @@ object BackendApp {
       val downloadAll = downloadAllImpl
       val addons = addonsImpl
       val attachment = attachImpl
+      val search = searchImpl
 
       def commands(eventContext: Option[EventContext]) =
         BackendCommands.fromBackend(this, eventContext)

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -181,7 +181,7 @@ object OFulltext {
           q = Query
             .all(account)
             .withFix(_.copy(query = itemIdsQuery.some))
-          res <- store.transact(QItem.searchStats(now.toUtcDate)(q))
+          res <- store.transact(QItem.searchStats(now.toUtcDate, None)(q))
         } yield res
       }
 
@@ -242,7 +242,7 @@ object OFulltext {
             .getOrElse(Attr.ItemId.notExists)
           qnext = q.withFix(_.copy(query = itemIdsQuery.some))
           now <- Timestamp.current[F]
-          res <- store.transact(QItem.searchStats(now.toUtcDate)(qnext))
+          res <- store.transact(QItem.searchStats(now.toUtcDate, None)(qnext))
         } yield res
 
       // Helper

modules/backend/src/main/scala/docspell/backend/ops/OItemSearch.scala

@@ -180,7 +180,7 @@ object OItemSearch {
         Timestamp
           .current[F]
           .map(_.toUtcDate)
-          .flatMap(today => store.transact(QItem.searchStats(today)(q)))
+          .flatMap(today => store.transact(QItem.searchStats(today, None)(q)))
 
       def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
         store

modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops.search
+
+import java.time.LocalDate
+
+import cats.effect._
+import cats.syntax.all._
+import cats.~>
+import fs2.Stream
+
+import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary}
+import docspell.common.{AccountId, SearchMode}
+import docspell.ftsclient.{FtsClient, FtsQuery}
+import docspell.query.{FulltextExtract, ItemQuery, ItemQueryParser}
+import docspell.store.Store
+import docspell.store.impl.TempFtsTable
+import docspell.store.qb.Batch
+import docspell.store.queries._
+
+import doobie.{ConnectionIO, WeakAsync}
+
+/** Combine fulltext search and sql search into one operation.
+  *
+  * To allow for paging the results from fulltext search are brought into the sql database
+  * by creating a temporary table.
+  */
+trait OSearch[F[_]] {
+
+  /** Searches at sql database with the given query joining it optionally with results
+    * from fulltext search. Any "fulltext search" query node is discarded. It is assumed
+    * that the fulltext search node has been extracted into the argument.
+    */
+  def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
+      q: Query,
+      fulltextQuery: Option[String]
+  ): F[Vector[ListItem]]
+
+  /** Same as `search` above, but runs additionally queries per item (!) to retrieve more
+    * details.
+    */
+  def searchWithDetails(
+      maxNoteLen: Int,
+      today: LocalDate,
+      batch: Batch
+  )(
+      q: Query,
+      fulltextQuery: Option[String]
+  ): F[Vector[ListItemWithTags]]
+
+  /** Run multiple database calls with the give query to collect a summary. */
+  def searchSummary(
+      mode: SearchMode,
+      today: LocalDate
+  )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
+
+  /** Parses a query string and creates a `Query` object, to be used with the other
+    * methods. The query object contains the parsed query amended with more conditions to
+    * restrict to valid items only (as specified with `mode`).
+    */
+  def parseQueryString(
+      accountId: AccountId,
+      mode: SearchMode,
+      qs: String
+  ): QueryParseResult
+}
+
+object OSearch {
+  def apply[F[_]: Async](
+      store: Store[F],
+      ftsClient: FtsClient[F]
+  ): OSearch[F] =
+    new OSearch[F] {
+
+      def parseQueryString(
+          accountId: AccountId,
+          mode: SearchMode,
+          qs: String
+      ): QueryParseResult =
+        ItemQueryParser.parse(qs) match {
+          case Right(iq) =>
+            val validItemQuery =
+              mode match {
+                case SearchMode.Trashed => ItemQuery.Expr.Trashed
+                case SearchMode.Normal  => ItemQuery.Expr.ValidItemStates
+                case SearchMode.All     => ItemQuery.Expr.ValidItemsOrTrashed
+              }
+            FulltextExtract.findFulltext(iq.expr) match {
+              case FulltextExtract.Result.SuccessNoFulltext(expr) =>
+                val qf = Query.Fix(accountId, Some(validItemQuery), None)
+                val qq = Query.QueryExpr(expr)
+                val q = Query(qf, qq)
+                QueryParseResult.Success(q, None)
+
+              case FulltextExtract.Result.SuccessNoExpr(fts) =>
+                val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
+                val qq = Query.QueryExpr(None)
+                val q = Query(qf, qq)
+                QueryParseResult.Success(q, Some(fts))
+
+              case FulltextExtract.Result.SuccessBoth(expr, fts) =>
+                val qf = Query.Fix(accountId, Some(validItemQuery), None)
+                val qq = Query.QueryExpr(expr)
+                val q = Query(qf, qq)
+                QueryParseResult.Success(q, Some(fts))
+
+              case f: FulltextExtract.FailureResult =>
+                QueryParseResult.FulltextMismatch(f)
+            }
+
+          case Left(err) =>
+            QueryParseResult.ParseFailed(err).cast
+        }
+
+      def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
+          q: Query,
+          fulltextQuery: Option[String]
+      ): F[Vector[ListItem]] =
+        fulltextQuery match {
+          case Some(ftq) =>
+            for {
+              ftq <- createFtsQuery(q.fix.account, batch, ftq)
+
+              results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
+                val tempTable = temporaryFtsTable(ftq, nat)
+                store
+                  .transact(
+                    Stream
+                      .eval(tempTable)
+                      .flatMap(tt =>
+                        QItem.queryItems(q, today, maxNoteLen, batch, tt.some)
+                      )
+                  )
+                  .compile
+                  .toVector
+              }
+
+            } yield results
+
+          case None =>
+            store
+              .transact(QItem.queryItems(q, today, maxNoteLen, batch, None))
+              .compile
+              .toVector
+        }
+
+      def searchWithDetails(
+          maxNoteLen: Int,
+          today: LocalDate,
+          batch: Batch
+      )(
+          q: Query,
+          fulltextQuery: Option[String]
+      ): F[Vector[ListItemWithTags]] =
+        for {
+          items <- search(maxNoteLen, today, batch)(q, fulltextQuery)
+          resolved <- store
+            .transact(
+              QItem.findItemsWithTags(q.fix.account.collective, Stream.emits(items))
+            )
+            .compile
+            .toVector
+        } yield resolved
+
+      def searchSummary(
+          mode: SearchMode,
+          today: LocalDate
+      )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
+        fulltextQuery match {
+          case Some(ftq) =>
+            for {
+              ftq <- createFtsQuery(q.fix.account, Batch.limit(500), ftq)
+              results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
+                val tempTable = temporaryFtsTable(ftq, nat)
+                store.transact(
+                  tempTable.flatMap(tt => QItem.searchStats(today, tt.some)(q))
+                )
+              }
+            } yield results
+
+          case None =>
+            store.transact(QItem.searchStats(today, None)(q))
+        }
+
+      private def createFtsQuery(
+          account: AccountId,
+          batch: Batch,
+          ftq: String
+      ): F[FtsQuery] =
+        store
+          .transact(QFolder.getMemberFolders(account))
+          .map(folders =>
+            FtsQuery(ftq, account.collective, batch.limit, batch.offset)
+              .withFolders(folders)
+          )
+
+      def temporaryFtsTable(
+          ftq: FtsQuery,
+          nat: F ~> ConnectionIO
+      ): ConnectionIO[TempFtsTable.Table] =
+        ftsClient
+          .searchAll(ftq)
+          .translate(nat)
+          .through(TempFtsTable.prepareTable(store.dbms, "fts_result"))
+          .compile
+          .lastOrError
+    }
+}

modules/backend/src/main/scala/docspell/backend/ops/search/QueryParseResult.scala

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops.search
+
+import docspell.query.{FulltextExtract, ParseFailure}
+import docspell.store.queries.Query
+
+sealed trait QueryParseResult {
+  def cast: QueryParseResult = this
+
+  def get: Option[(Query, Option[String])]
+  def isSuccess: Boolean = get.isDefined
+  def isFailure: Boolean = !isSuccess
+}
+
+object QueryParseResult {
+
+  final case class Success(q: Query, ftq: Option[String]) extends QueryParseResult {
+    val get = Some(q -> ftq)
+  }
+
+  final case class ParseFailed(error: ParseFailure) extends QueryParseResult {
+    val get = None
+  }
+
+  final case class FulltextMismatch(error: FulltextExtract.FailureResult)
+      extends QueryParseResult {
+    val get = None
+  }
+}

modules/fts-client/src/main/scala/docspell/ftsclient/FtsQuery.scala

@@ -37,6 +37,8 @@ final case class FtsQuery(
 }
 
 object FtsQuery {
+  def apply(q: String, collective: Ident, limit: Int, offset: Int): FtsQuery =
+    FtsQuery(q, collective, Set.empty, Set.empty, limit, offset, HighlightSetting.default)
 
   case class HighlightSetting(pre: String, post: String)
 

modules/notification/impl/src/main/scala/docspell/notification/impl/context/BasicData.scala

@@ -79,7 +79,7 @@ object BasicData {
         Query.Fix(
           account,
           Some(ItemQuery.Attr.ItemId.in(itemIds.map(_.id))),
-          Some(_.created)
+          Some(_.byItemColumnAsc(_.created))
         )
       )
       for {

modules/restserver/src/main/resources/reference.conf

@@ -70,6 +70,8 @@ docspell.server {
   # In order to keep this low, a limit can be defined here.
   max-note-length = 180
 
+  feature-search-2 = true
+
 
   # This defines whether the classification form in the collective
   # settings is displayed or not. If all joex instances have document

modules/restserver/src/main/scala/docspell/restserver/Config.scala

@@ -34,6 +34,7 @@ case class Config(
     integrationEndpoint: Config.IntegrationEndpoint,
     maxItemPageSize: Int,
     maxNoteLength: Int,
+    featureSearch2: Boolean,
     fullTextSearch: Config.FullTextSearch,
     adminEndpoint: Config.AdminEndpoint,
     openid: List[OpenIdConfig],

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -35,7 +35,10 @@ import org.http4s.server.websocket.WebSocketBuilder2
 
 object RestServer {
 
-  def serve[F[_]: Async](cfg: Config, pools: Pools): F[ExitCode] =
+  def serve[F[_]: Async](
+      cfg: Config,
+      pools: Pools
+  ): F[ExitCode] =
     for {
       wsTopic <- Topic[F, OutputEvent]
       keepAlive = Stream
@@ -102,7 +105,8 @@ object RestServer {
         cfg.auth.serverSecret.some
       )
 
-      restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic)
+      restApp <- RestAppImpl
+        .create[F](cfg, pools, store, httpClient, pubSub, wsTopic)
     } yield (restApp, pubSub, setting)
 
   def createHttpApp[F[_]: Async](

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -44,386 +44,393 @@ object ItemRoutes {
     val dsl = new Http4sDsl[F] {}
     import dsl._
 
-    HttpRoutes.of {
+    ItemSearchPart(backend, cfg, user) <+>
-      case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
+      HttpRoutes.of {
-            offset
+        case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
-          ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
+              offset
-        val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
+            ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
-          .restrictLimitTo(cfg.maxItemPageSize)
+          val batch = Batch(offset.getOrElse(0), limit.getOrElse(cfg.maxItemPageSize))
-        val limitCapped = limit.exists(_ > cfg.maxItemPageSize)
+            .restrictLimitTo(cfg.maxItemPageSize)
-        val itemQuery = ItemQueryString(q)
+          val limitCapped = limit.exists(_ > cfg.maxItemPageSize)
-        val settings = OSimpleSearch.Settings(
+          val itemQuery = ItemQueryString(q)
-          batch,
+          val settings = OSimpleSearch.Settings(
-          cfg.fullTextSearch.enabled,
-          detailFlag.getOrElse(false),
-          cfg.maxNoteLength,
-          searchMode.getOrElse(SearchMode.Normal)
-        )
-        val fixQuery = Query.Fix(user.account, None, None)
-        searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
-
-      case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
-        val itemQuery = ItemQueryString(q)
-        val fixQuery = Query.Fix(user.account, None, None)
-        val settings = OSimpleSearch.StatsSettings(
-          useFTS = cfg.fullTextSearch.enabled,
-          searchMode = searchMode.getOrElse(SearchMode.Normal)
-        )
-        searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
-
-      case req @ POST -> Root / "search" =>
-        for {
-          userQuery <- req.as[ItemQuery]
-          batch = Batch(
-            userQuery.offset.getOrElse(0),
-            userQuery.limit.getOrElse(cfg.maxItemPageSize)
-          ).restrictLimitTo(
-            cfg.maxItemPageSize
-          )
-          limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
-          itemQuery = ItemQueryString(userQuery.query)
-          settings = OSimpleSearch.Settings(
             batch,
             cfg.fullTextSearch.enabled,
-            userQuery.withDetails.getOrElse(false),
+            detailFlag.getOrElse(false),
             cfg.maxNoteLength,
-            searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+            searchMode.getOrElse(SearchMode.Normal)
           )
-          fixQuery = Query.Fix(user.account, None, None)
+          val fixQuery = Query.Fix(user.account, None, None)
-          resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
+          searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
-        } yield resp
 
-      case req @ POST -> Root / "searchStats" =>
+        case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
-        for {
+          val itemQuery = ItemQueryString(q)
-          userQuery <- req.as[ItemQuery]
+          val fixQuery = Query.Fix(user.account, None, None)
-          itemQuery = ItemQueryString(userQuery.query)
+          val settings = OSimpleSearch.StatsSettings(
-          fixQuery = Query.Fix(user.account, None, None)
-          settings = OSimpleSearch.StatsSettings(
             useFTS = cfg.fullTextSearch.enabled,
-            searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+            searchMode = searchMode.getOrElse(SearchMode.Normal)
           )
-          resp <- searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
+          searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
-        } yield resp
 
-      case req @ POST -> Root / "searchIndex" =>
+        case req @ POST -> Root / "search" =>
-        for {
+          for {
-          mask <- req.as[ItemQuery]
+            userQuery <- req.as[ItemQuery]
-          limitCapped = mask.limit.exists(_ > cfg.maxItemPageSize)
+            batch = Batch(
-          resp <- mask.query match {
+              userQuery.offset.getOrElse(0),
-            case q if q.length > 1 =>
+              userQuery.limit.getOrElse(cfg.maxItemPageSize)
-              val ftsIn = OFulltext.FtsInput(q)
+            ).restrictLimitTo(
-              val batch = Batch(
+              cfg.maxItemPageSize
-                mask.offset.getOrElse(0),
+            )
-                mask.limit.getOrElse(cfg.maxItemPageSize)
+            limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
-              ).restrictLimitTo(cfg.maxItemPageSize)
+            itemQuery = ItemQueryString(userQuery.query)
-              for {
+            settings = OSimpleSearch.Settings(
-                items <- backend.fulltext
+              batch,
-                  .findIndexOnly(cfg.maxNoteLength)(ftsIn, user.account, batch)
+              cfg.fullTextSearch.enabled,
-                ok <- Ok(
+              userQuery.withDetails.getOrElse(false),
-                  Conversions.mkItemListWithTagsFtsPlain(items, batch, limitCapped)
+              cfg.maxNoteLength,
+              searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+            )
+            fixQuery = Query.Fix(user.account, None, None)
+            resp <- searchItems(backend, dsl)(settings, fixQuery, itemQuery, limitCapped)
+          } yield resp
+
+        case req @ POST -> Root / "searchStats" =>
+          for {
+            userQuery <- req.as[ItemQuery]
+            itemQuery = ItemQueryString(userQuery.query)
+            fixQuery = Query.Fix(user.account, None, None)
+            settings = OSimpleSearch.StatsSettings(
+              useFTS = cfg.fullTextSearch.enabled,
+              searchMode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+            )
+            resp <- searchItemStats(backend, dsl)(settings, fixQuery, itemQuery)
+          } yield resp
+
+        case req @ POST -> Root / "searchIndex" =>
+          for {
+            mask <- req.as[ItemQuery]
+            limitCapped = mask.limit.exists(_ > cfg.maxItemPageSize)
+            resp <- mask.query match {
+              case q if q.length > 1 =>
+                val ftsIn = OFulltext.FtsInput(q)
+                val batch = Batch(
+                  mask.offset.getOrElse(0),
+                  mask.limit.getOrElse(cfg.maxItemPageSize)
+                ).restrictLimitTo(cfg.maxItemPageSize)
+                for {
+                  items <- backend.fulltext
+                    .findIndexOnly(cfg.maxNoteLength)(ftsIn, user.account, batch)
+                  ok <- Ok(
+                    Conversions.mkItemListWithTagsFtsPlain(items, batch, limitCapped)
+                  )
+                } yield ok
+
+              case _ =>
+                BadRequest(BasicResult(false, "Query string too short"))
+            }
+          } yield resp
+
+        case GET -> Root / Ident(id) =>
+          for {
+            item <- backend.itemSearch.findItem(id, user.account.collective)
+            result = item.map(Conversions.mkItemDetail)
+            resp <-
+              result
+                .map(r => Ok(r))
+                .getOrElse(NotFound(BasicResult(false, "Not found.")))
+          } yield resp
+
+        case POST -> Root / Ident(id) / "confirm" =>
+          for {
+            res <- backend.item.setState(id, ItemState.Confirmed, user.account.collective)
+            resp <- Ok(Conversions.basicResult(res, "Item data confirmed"))
+          } yield resp
+
+        case POST -> Root / Ident(id) / "unconfirm" =>
+          for {
+            res <- backend.item.setState(id, ItemState.Created, user.account.collective)
+            resp <- Ok(Conversions.basicResult(res, "Item back to created."))
+          } yield resp
+
+        case POST -> Root / Ident(id) / "restore" =>
+          for {
+            res <- backend.item.restore(NonEmptyList.of(id), user.account.collective)
+            resp <- Ok(Conversions.basicResult(res, "Item restored."))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "tags" =>
+          for {
+            tags <- req.as[StringList].map(_.items)
+            res <- backend.item.setTags(id, tags, user.account.collective)
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Tags updated"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "tags" =>
+          for {
+            data <- req.as[Tag]
+            rtag <- Conversions.newTag(data, user.account.collective)
+            res <- backend.item.addNewTag(user.account.collective, id, rtag)
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Tag added."))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "taglink" =>
+          for {
+            tags <- req.as[StringList]
+            res <- backend.item.linkTags(id, tags.items, user.account.collective)
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Tags linked"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "tagtoggle" =>
+          for {
+            tags <- req.as[StringList]
+            res <- backend.item.toggleTags(id, tags.items, user.account.collective)
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Tags linked"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "tagsremove" =>
+          for {
+            json <- req.as[StringList]
+            res <- backend.item.removeTagsMultipleItems(
+              NonEmptyList.of(id),
+              json.items,
+              user.account.collective
+            )
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Tags removed"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "direction" =>
+          for {
+            dir <- req.as[DirectionValue]
+            res <- backend.item.setDirection(
+              NonEmptyList.of(id),
+              dir.direction,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Direction updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "folder" =>
+          for {
+            idref <- req.as[OptionalId]
+            res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective)
+            resp <- Ok(Conversions.basicResult(res, "Folder updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "corrOrg" =>
+          for {
+            idref <- req.as[OptionalId]
+            res <- backend.item.setCorrOrg(
+              NonEmptyList.of(id),
+              idref.id,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "corrOrg" =>
+          for {
+            data <- req.as[Organization]
+            org <- Conversions.newOrg(data, user.account.collective)
+            res <- backend.item.addCorrOrg(id, org)
+            resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "corrPerson" =>
+          for {
+            idref <- req.as[OptionalId]
+            res <- backend.item.setCorrPerson(
+              NonEmptyList.of(id),
+              idref.id,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "corrPerson" =>
+          for {
+            data <- req.as[Person]
+            pers <- Conversions.newPerson(data, user.account.collective)
+            res <- backend.item.addCorrPerson(id, pers)
+            resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "concPerson" =>
+          for {
+            idref <- req.as[OptionalId]
+            res <- backend.item.setConcPerson(
+              NonEmptyList.of(id),
+              idref.id,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "concPerson" =>
+          for {
+            data <- req.as[Person]
+            pers <- Conversions.newPerson(data, user.account.collective)
+            res <- backend.item.addConcPerson(id, pers)
+            resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "concEquipment" =>
+          for {
+            idref <- req.as[OptionalId]
+            res <- backend.item.setConcEquip(
+              NonEmptyList.of(id),
+              idref.id,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "concEquipment" =>
+          for {
+            data <- req.as[Equipment]
+            equip <- Conversions.newEquipment(data, user.account.collective)
+            res <- backend.item.addConcEquip(id, equip)
+            resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "notes" =>
+          for {
+            text <- req.as[OptionalText]
+            res <- backend.item.setNotes(id, text.text.notEmpty, user.account.collective)
+            resp <- Ok(Conversions.basicResult(res, "Notes updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "name" =>
+          for {
+            text <- req.as[OptionalText]
+            res <- backend.item.setName(
+              id,
+              text.text.notEmpty.getOrElse(""),
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Name updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "duedate" =>
+          for {
+            date <- req.as[OptionalDate]
+            _ <- logger.debug(s"Setting item due date to ${date.date}")
+            res <- backend.item.setItemDueDate(
+              NonEmptyList.of(id),
+              date.date,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
+          } yield resp
+
+        case req @ PUT -> Root / Ident(id) / "date" =>
+          for {
+            date <- req.as[OptionalDate]
+            _ <- logger.debug(s"Setting item date to ${date.date}")
+            res <- backend.item.setItemDate(
+              NonEmptyList.of(id),
+              date.date,
+              user.account.collective
+            )
+            resp <- Ok(Conversions.basicResult(res, "Item date updated"))
+          } yield resp
+
+        case GET -> Root / Ident(id) / "proposals" =>
+          for {
+            ml <- backend.item.getProposals(id, user.account.collective)
+            ip = Conversions.mkItemProposals(ml)
+            resp <- Ok(ip)
+          } yield resp
+
+        case req @ POST -> Root / Ident(id) / "attachment" / "movebefore" =>
+          for {
+            data <- req.as[MoveAttachment]
+            _ <- logger.debug(s"Move item (${id.id}) attachment $data")
+            res <- backend.item.moveAttachmentBefore(id, data.source, data.target)
+            resp <- Ok(Conversions.basicResult(res, "Attachment moved."))
+          } yield resp
+
+        case req @ GET -> Root / Ident(id) / "preview" :? QP.WithFallback(flag) =>
+          def notFound =
+            NotFound(BasicResult(false, "Not found"))
+          for {
+            preview <- backend.itemSearch.findItemPreview(id, user.account.collective)
+            inm = req.headers.get[`If-None-Match`].flatMap(_.tags)
+            matches = BinaryUtil.matchETag(preview.map(_.meta), inm)
+            fallback = flag.getOrElse(false)
+            resp <-
+              preview
+                .map { data =>
+                  if (matches) BinaryUtil.withResponseHeaders(dsl, NotModified())(data)
+                  else BinaryUtil.makeByteResp(dsl)(data).map(Responses.noCache)
+                }
+                .getOrElse(
+                  if (fallback) BinaryUtil.noPreview(req.some).getOrElseF(notFound)
+                  else notFound
                 )
-              } yield ok
+          } yield resp
 
-            case _ =>
+        case HEAD -> Root / Ident(id) / "preview" =>
-              BadRequest(BasicResult(false, "Query string too short"))
+          for {
-          }
+            preview <- backend.itemSearch.findItemPreview(id, user.account.collective)
-        } yield resp
+            resp <-
+              preview
+                .map(data => BinaryUtil.withResponseHeaders(dsl, Ok())(data))
+                .getOrElse(NotFound(BasicResult(false, "Not found")))
+          } yield resp
 
-      case GET -> Root / Ident(id) =>
+        case req @ POST -> Root / Ident(id) / "reprocess" =>
-        for {
+          for {
-          item <- backend.itemSearch.findItem(id, user.account.collective)
+            data <- req.as[IdList]
-          result = item.map(Conversions.mkItemDetail)
+            _ <- logger.debug(s"Re-process item ${id.id}")
-          resp <-
+            res <- backend.item.reprocess(id, data.ids, user.account)
-            result
+            resp <- Ok(Conversions.basicResult(res, "Re-process task submitted."))
-              .map(r => Ok(r))
+          } yield resp
-              .getOrElse(NotFound(BasicResult(false, "Not found.")))
-        } yield resp
 
-      case POST -> Root / Ident(id) / "confirm" =>
+        case req @ PUT -> Root / Ident(id) / "customfield" =>
-        for {
+          for {
-          res <- backend.item.setState(id, ItemState.Confirmed, user.account.collective)
+            data <- req.as[CustomFieldValue]
-          resp <- Ok(Conversions.basicResult(res, "Item data confirmed"))
+            res <- backend.customFields.setValue(
-        } yield resp
+              id,
+              SetValue(data.field, data.value, user.account.collective)
+            )
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value))
+          } yield resp
 
-      case POST -> Root / Ident(id) / "unconfirm" =>
+        case req @ DELETE -> Root / Ident(id) / "customfield" / Ident(fieldId) =>
-        for {
+          for {
-          res <- backend.item.setState(id, ItemState.Created, user.account.collective)
+            res <- backend.customFields.deleteValue(
-          resp <- Ok(Conversions.basicResult(res, "Item back to created."))
+              RemoveValue(fieldId, NonEmptyList.of(id), user.account.collective)
-        } yield resp
+            )
+            baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+            resp <- Ok(Conversions.basicResult(res.value, "Custom field value removed."))
+          } yield resp
 
-      case POST -> Root / Ident(id) / "restore" =>
+        case DELETE -> Root / Ident(id) =>
-        for {
+          for {
-          res <- backend.item.restore(NonEmptyList.of(id), user.account.collective)
+            n <- backend.item.setDeletedState(
-          resp <- Ok(Conversions.basicResult(res, "Item restored."))
+              NonEmptyList.of(id),
-        } yield resp
+              user.account.collective
-
+            )
-      case req @ PUT -> Root / Ident(id) / "tags" =>
+            res = BasicResult(
-        for {
+              n > 0,
-          tags <- req.as[StringList].map(_.items)
+              if (n > 0) "Item deleted" else "Item deletion failed."
-          res <- backend.item.setTags(id, tags, user.account.collective)
+            )
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
+            resp <- Ok(res)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
+          } yield resp
-          resp <- Ok(Conversions.basicResult(res.value, "Tags updated"))
+      }
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "tags" =>
-        for {
-          data <- req.as[Tag]
-          rtag <- Conversions.newTag(data, user.account.collective)
-          res <- backend.item.addNewTag(user.account.collective, id, rtag)
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value, "Tag added."))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "taglink" =>
-        for {
-          tags <- req.as[StringList]
-          res <- backend.item.linkTags(id, tags.items, user.account.collective)
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value, "Tags linked"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "tagtoggle" =>
-        for {
-          tags <- req.as[StringList]
-          res <- backend.item.toggleTags(id, tags.items, user.account.collective)
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value, "Tags linked"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "tagsremove" =>
-        for {
-          json <- req.as[StringList]
-          res <- backend.item.removeTagsMultipleItems(
-            NonEmptyList.of(id),
-            json.items,
-            user.account.collective
-          )
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value, "Tags removed"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "direction" =>
-        for {
-          dir <- req.as[DirectionValue]
-          res <- backend.item.setDirection(
-            NonEmptyList.of(id),
-            dir.direction,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Direction updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "folder" =>
-        for {
-          idref <- req.as[OptionalId]
-          res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective)
-          resp <- Ok(Conversions.basicResult(res, "Folder updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "corrOrg" =>
-        for {
-          idref <- req.as[OptionalId]
-          res <- backend.item.setCorrOrg(
-            NonEmptyList.of(id),
-            idref.id,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "corrOrg" =>
-        for {
-          data <- req.as[Organization]
-          org <- Conversions.newOrg(data, user.account.collective)
-          res <- backend.item.addCorrOrg(id, org)
-          resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "corrPerson" =>
-        for {
-          idref <- req.as[OptionalId]
-          res <- backend.item.setCorrPerson(
-            NonEmptyList.of(id),
-            idref.id,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "corrPerson" =>
-        for {
-          data <- req.as[Person]
-          pers <- Conversions.newPerson(data, user.account.collective)
-          res <- backend.item.addCorrPerson(id, pers)
-          resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "concPerson" =>
-        for {
-          idref <- req.as[OptionalId]
-          res <- backend.item.setConcPerson(
-            NonEmptyList.of(id),
-            idref.id,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "concPerson" =>
-        for {
-          data <- req.as[Person]
-          pers <- Conversions.newPerson(data, user.account.collective)
-          res <- backend.item.addConcPerson(id, pers)
-          resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "concEquipment" =>
-        for {
-          idref <- req.as[OptionalId]
-          res <- backend.item.setConcEquip(
-            NonEmptyList.of(id),
-            idref.id,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "concEquipment" =>
-        for {
-          data <- req.as[Equipment]
-          equip <- Conversions.newEquipment(data, user.account.collective)
-          res <- backend.item.addConcEquip(id, equip)
-          resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "notes" =>
-        for {
-          text <- req.as[OptionalText]
-          res <- backend.item.setNotes(id, text.text.notEmpty, user.account.collective)
-          resp <- Ok(Conversions.basicResult(res, "Notes updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "name" =>
-        for {
-          text <- req.as[OptionalText]
-          res <- backend.item.setName(
-            id,
-            text.text.notEmpty.getOrElse(""),
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Name updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "duedate" =>
-        for {
-          date <- req.as[OptionalDate]
-          _ <- logger.debug(s"Setting item due date to ${date.date}")
-          res <- backend.item.setItemDueDate(
-            NonEmptyList.of(id),
-            date.date,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "date" =>
-        for {
-          date <- req.as[OptionalDate]
-          _ <- logger.debug(s"Setting item date to ${date.date}")
-          res <- backend.item.setItemDate(
-            NonEmptyList.of(id),
-            date.date,
-            user.account.collective
-          )
-          resp <- Ok(Conversions.basicResult(res, "Item date updated"))
-        } yield resp
-
-      case GET -> Root / Ident(id) / "proposals" =>
-        for {
-          ml <- backend.item.getProposals(id, user.account.collective)
-          ip = Conversions.mkItemProposals(ml)
-          resp <- Ok(ip)
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "attachment" / "movebefore" =>
-        for {
-          data <- req.as[MoveAttachment]
-          _ <- logger.debug(s"Move item (${id.id}) attachment $data")
-          res <- backend.item.moveAttachmentBefore(id, data.source, data.target)
-          resp <- Ok(Conversions.basicResult(res, "Attachment moved."))
-        } yield resp
-
-      case req @ GET -> Root / Ident(id) / "preview" :? QP.WithFallback(flag) =>
-        def notFound =
-          NotFound(BasicResult(false, "Not found"))
-        for {
-          preview <- backend.itemSearch.findItemPreview(id, user.account.collective)
-          inm = req.headers.get[`If-None-Match`].flatMap(_.tags)
-          matches = BinaryUtil.matchETag(preview.map(_.meta), inm)
-          fallback = flag.getOrElse(false)
-          resp <-
-            preview
-              .map { data =>
-                if (matches) BinaryUtil.withResponseHeaders(dsl, NotModified())(data)
-                else BinaryUtil.makeByteResp(dsl)(data).map(Responses.noCache)
-              }
-              .getOrElse(
-                if (fallback) BinaryUtil.noPreview(req.some).getOrElseF(notFound)
-                else notFound
-              )
-        } yield resp
-
-      case HEAD -> Root / Ident(id) / "preview" =>
-        for {
-          preview <- backend.itemSearch.findItemPreview(id, user.account.collective)
-          resp <-
-            preview
-              .map(data => BinaryUtil.withResponseHeaders(dsl, Ok())(data))
-              .getOrElse(NotFound(BasicResult(false, "Not found")))
-        } yield resp
-
-      case req @ POST -> Root / Ident(id) / "reprocess" =>
-        for {
-          data <- req.as[IdList]
-          _ <- logger.debug(s"Re-process item ${id.id}")
-          res <- backend.item.reprocess(id, data.ids, user.account)
-          resp <- Ok(Conversions.basicResult(res, "Re-process task submitted."))
-        } yield resp
-
-      case req @ PUT -> Root / Ident(id) / "customfield" =>
-        for {
-          data <- req.as[CustomFieldValue]
-          res <- backend.customFields.setValue(
-            id,
-            SetValue(data.field, data.value, user.account.collective)
-          )
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value))
-        } yield resp
-
-      case req @ DELETE -> Root / Ident(id) / "customfield" / Ident(fieldId) =>
-        for {
-          res <- backend.customFields.deleteValue(
-            RemoveValue(fieldId, NonEmptyList.of(id), user.account.collective)
-          )
-          baseUrl = ClientRequestInfo.getBaseUrl(cfg, req)
-          _ <- backend.notification.offerEvents(res.event(user.account, baseUrl.some))
-          resp <- Ok(Conversions.basicResult(res.value, "Custom field value removed."))
-        } yield resp
-
-      case DELETE -> Root / Ident(id) =>
-        for {
-          n <- backend.item.setDeletedState(NonEmptyList.of(id), user.account.collective)
-          res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
-          resp <- Ok(res)
-        } yield resp
-    }
   }
 
   def searchItems[F[_]: Sync](

modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver.routes
+
+import cats.effect.Async
+
+import docspell.backend.BackendApp
+import docspell.backend.auth.AuthToken
+import docspell.restserver.Config
+import docspell.restserver.http4s.{QueryParam => QP}
+
+import org.http4s.HttpRoutes
+import org.http4s.dsl.Http4sDsl
+
+@annotation.nowarn
+object ItemSearchPart {
+
+  def apply[F[_]: Async](
+      backend: BackendApp[F],
+      cfg: Config,
+      authToken: AuthToken
+  ): HttpRoutes[F] =
+    if (cfg.featureSearch2) routes(backend, cfg, authToken)
+    else HttpRoutes.empty
+
+  def routes[F[_]: Async](
+      backend: BackendApp[F],
+      cfg: Config,
+      authToken: AuthToken
+  ): HttpRoutes[F] = {
+    val logger = docspell.logging.getLogger[F]
+    val dsl = new Http4sDsl[F] {}
+    import dsl._
+
+    HttpRoutes.of {
+      case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
+            offset
+          ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
+        ???
+
+      case req @ POST -> Root / "search" =>
+        ???
+
+      case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
+        ???
+
+      case req @ POST -> Root / "searchStats" =>
+        ???
+    }
+  }
+}

modules/store/src/main/scala/docspell/store/impl/TempFtsTable.scala

@@ -0,0 +1,213 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.impl
+
+import cats.Foldable
+import cats.data.NonEmptyList
+import cats.effect._
+import cats.syntax.all._
+import fs2.{Pipe, Stream}
+
+import docspell.common.{Duration, Ident}
+import docspell.ftsclient.FtsResult
+import docspell.store.Db
+import docspell.store.qb.DSL._
+import docspell.store.qb._
+
+import doobie._
+import doobie.implicits._
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Temporary table used to store item ids fetched from fulltext search */
+object TempFtsTable {
+  private[this] val logger = docspell.logging.getLogger[ConnectionIO]
+
+  case class Row(id: Ident, score: Option[Double], context: Option[ContextEntry])
+  object Row {
+    def from(result: FtsResult)(m: FtsResult.ItemMatch): Row = {
+      val context = m.data match {
+        case FtsResult.AttachmentData(_, attachName) =>
+          result.highlight
+            .get(m.id)
+            .filter(_.nonEmpty)
+            .map(str => ContextEntry(attachName, str))
+
+        case FtsResult.ItemData =>
+          result.highlight
+            .get(m.id)
+            .filter(_.nonEmpty)
+            .map(str => ContextEntry("item", str))
+      }
+      Row(m.itemId, m.score.some, context)
+    }
+  }
+
+  case class ContextEntry(name: String, context: List[String])
+  object ContextEntry {
+    implicit val jsonDecoder: Decoder[ContextEntry] = deriveDecoder
+    implicit val jsonEncoder: Encoder[ContextEntry] = deriveEncoder
+
+    implicit val meta: Meta[ContextEntry] =
+      jsonMeta[ContextEntry]
+  }
+
+  case class Table(tableName: String, alias: Option[String], dbms: Db) extends TableDef {
+    val id: Column[Ident] = Column("id", this)
+    val score: Column[Double] = Column("score", this)
+    val context: Column[ContextEntry] = Column("context", this)
+
+    val all: NonEmptyList[Column[_]] = NonEmptyList.of(id, score, context)
+
+    def as(newAlias: String): Table = copy(alias = Some(newAlias))
+
+    def distinctCte(name: String) =
+      dbms.fold(
+        TempFtsTable.distinctCtePg(this, name),
+        TempFtsTable.distinctCteMaria(this, name),
+        TempFtsTable.distinctCteH2(this, name)
+      )
+
+    def distinctCteSimple(name: String) =
+      CteBind(copy(tableName = name) -> Select(select(id), from(this)).distinct)
+
+    def insertAll[F[_]: Foldable](rows: F[Row]): ConnectionIO[Int] =
+      insertBatch(this, rows)
+
+    def dropTable: ConnectionIO[Int] =
+      TempFtsTable.dropTable(Fragment.const0(tableName)).update.run
+
+    def createIndex: ConnectionIO[Unit] = {
+      val analyze = dbms.fold(
+        TempFtsTable.analyzeTablePg(this),
+        cio.unit,
+        cio.unit
+      )
+
+      TempFtsTable.createIndex(this) *> analyze
+    }
+
+    def insert: Pipe[ConnectionIO, FtsResult, Int] =
+      in => in.evalMap(res => insertAll(res.results.map(Row.from(res))))
+  }
+
+  def createTable(db: Db, name: String): ConnectionIO[Table] = {
+    val stmt = db.fold(
+      createTablePostgreSQL(Fragment.const(name)),
+      createTableMariaDB(Fragment.const0(name)),
+      createTableH2(Fragment.const0(name))
+    )
+    stmt.as(Table(name, None, db))
+  }
+
+  def prepareTable(db: Db, name: String): Pipe[ConnectionIO, FtsResult, Table] =
+    in =>
+      for {
+        timed <- Stream.eval(Duration.stopTime[ConnectionIO])
+        tt <- Stream.eval(createTable(db, name))
+        n <- in.through(tt.insert).foldMonoid
+        _ <- Stream.eval(tt.createIndex)
+        duration <- Stream.eval(timed)
+        _ <- Stream.eval(
+          logger.info(
+            s"Creating temporary fts table ($n elements) took: ${duration.formatExact}"
+          )
+        )
+      } yield tt
+
+  private def dropTable(name: Fragment): Fragment =
+    sql"""DROP TABLE IF EXISTS $name"""
+
+  private def createTableH2(name: Fragment): ConnectionIO[Int] =
+    sql"""${dropTable(name)}; CREATE LOCAL TEMPORARY TABLE $name (
+         |  id varchar not null,
+         |  score double precision,
+         |  context text
+         |);""".stripMargin.update.run
+
+  private def createTableMariaDB(name: Fragment): ConnectionIO[Int] =
+    dropTable(name).update.run *>
+      sql"""CREATE TEMPORARY TABLE $name (
+           |  id varchar(254) not null,
+           |  score double,
+           |  context mediumtext
+           |)""".stripMargin.update.run
+
+  private def createTablePostgreSQL(name: Fragment): ConnectionIO[Int] =
+    sql"""CREATE TEMPORARY TABLE IF NOT EXISTS $name (
+         |  id varchar not null,
+         |  score double precision,
+         |  context text
+         |) ON COMMIT DROP;""".stripMargin.update.run
+
+  private def createIndex(table: Table): ConnectionIO[Unit] = {
+    val tableName = Fragment.const0(table.tableName)
+
+    val idIdxName = Fragment.const0(s"${table.tableName}_id_idx")
+    val id = Fragment.const0(table.id.name)
+    val scoreIdxName = Fragment.const0(s"${table.tableName}_score_idx")
+    val score = Fragment.const0(table.score.name)
+
+    sql"CREATE INDEX IF NOT EXISTS $idIdxName ON $tableName($id)".update.run.void *>
+      sql"CREATE INDEX IF NOT EXISTS $scoreIdxName ON $tableName($score)".update.run.void
+  }
+
+  private def analyzeTablePg(table: Table): ConnectionIO[Unit] = {
+    val tableName = Fragment.const0(table.tableName)
+    sql"ANALYZE $tableName".update.run.void
+  }
+
+  private def insertBatch[F[_]: Foldable](table: Table, rows: F[Row]) = {
+    val sql =
+      s"""INSERT INTO ${table.tableName}
+         |  (${table.id.name}, ${table.score.name}, ${table.context.name})
+         |  VALUES (?, ?, ?)""".stripMargin
+
+    Update[Row](sql).updateMany(rows)
+  }
+
+  private def distinctCtePg(table: Table, name: String): CteBind =
+    CteBind(
+      table.copy(tableName = name) ->
+        Select(
+          select(
+            table.id.s,
+            max(table.score).as(table.score.name),
+            rawFunction("string_agg", table.context.s, lit("','")).as(table.context.name)
+          ),
+          from(table)
+        ).groupBy(table.id)
+    )
+
+  private def distinctCteMaria(table: Table, name: String): CteBind =
+    CteBind(
+      table.copy(tableName = name) ->
+        Select(
+          select(
+            table.id.s,
+            max(table.score).as(table.score.name),
+            rawFunction("group_concat", table.context.s).as(table.context.name)
+          ),
+          from(table)
+        ).groupBy(table.id)
+    )
+
+  private def distinctCteH2(table: Table, name: String): CteBind =
+    CteBind(
+      table.copy(tableName = name) ->
+        Select(
+          select(
+            table.id.s,
+            max(table.score).as(table.score.name),
+            rawFunction("listagg", table.context.s, lit("','")).as(table.context.name)
+          ),
+          from(table)
+        ).groupBy(table.id)
+    )
+
+  private val cio: Sync[ConnectionIO] = Sync[ConnectionIO]
+}

modules/store/src/main/scala/docspell/store/impl/TempIdTable.scala

@@ -1,85 +0,0 @@
-package docspell.store.impl
-
-import cats.Foldable
-import cats.data.NonEmptyList
-import cats.effect._
-import cats.syntax.all._
-import docspell.common.Ident
-import docspell.store.Db
-import docspell.store.qb.{Column, TableDef}
-import docspell.store.impl.DoobieMeta._
-import doobie._
-import doobie.implicits._
-
-/** Temporary table used to store item ids fetched from fulltext search */
-object TempIdTable {
-  case class Row(id: Ident)
-  case class Table(tableName: String, alias: Option[String], dbms: Db) extends TableDef {
-    val id: Column[Ident] = Column("id", this)
-
-    val all: NonEmptyList[Column[_]] = NonEmptyList.of(id)
-
-    def as(newAlias: String): Table = copy(alias = Some(newAlias))
-
-    def insertAll[F[_]: Foldable](rows: F[Row]): ConnectionIO[Int] =
-      insertBatch(this, rows)
-
-    def dropTable: ConnectionIO[Int] =
-      TempIdTable.dropTable(Fragment.const0(tableName)).update.run
-
-    def createIndex: ConnectionIO[Unit] = {
-      val analyze = dbms.fold(
-        TempIdTable.analyzeTablePg(this),
-        Sync[ConnectionIO].unit,
-        Sync[ConnectionIO].unit
-      )
-
-      TempIdTable.createIndex(this) *> analyze
-    }
-  }
-
-  def createTable(db: Db, name: String): ConnectionIO[Table] = {
-    val stmt = db.fold(
-      createTablePostgreSQL(Fragment.const(name)),
-      createTableMariaDB(Fragment.const0(name)),
-      createTableH2(Fragment.const0(name))
-    )
-    stmt.as(Table(name, None, db))
-  }
-
-  private def dropTable(name: Fragment): Fragment =
-    sql"""DROP TABLE IF EXISTS $name"""
-
-  private def createTableH2(name: Fragment): ConnectionIO[Int] =
-    sql"""${dropTable(name)}; CREATE LOCAL TEMPORARY TABLE $name (
-         |  id varchar not null
-         |);""".stripMargin.update.run
-
-  private def createTableMariaDB(name: Fragment): ConnectionIO[Int] =
-    dropTable(name).update.run *>
-      sql"CREATE TEMPORARY TABLE $name (id varchar(254) not null);".update.run
-
-  private def createTablePostgreSQL(name: Fragment): ConnectionIO[Int] =
-    sql"""CREATE TEMPORARY TABLE IF NOT EXISTS $name (
-         |  id varchar not null
-         |) ON COMMIT DROP;""".stripMargin.update.run
-
-  private def createIndex(table: Table): ConnectionIO[Unit] = {
-    val idxName = Fragment.const0(s"${table.tableName}_id_idx")
-    val tableName = Fragment.const0(table.tableName)
-    val col = Fragment.const0(table.id.name)
-    sql"""CREATE INDEX IF NOT EXISTS $idxName ON $tableName($col);""".update.run.void
-  }
-
-  private def analyzeTablePg(table: Table): ConnectionIO[Unit] = {
-    val tableName = Fragment.const0(table.tableName)
-    sql"ANALYZE $tableName".update.run.void
-  }
-
-  private def insertBatch[F[_]: Foldable](table: Table, rows: F[Row]) = {
-    val sql =
-      s"INSERT INTO ${table.tableName} (${table.id.name}) VALUES (?)"
-
-    Update[Row](sql).updateMany(rows)
-  }
-}

modules/store/src/main/scala/docspell/store/qb/DBFunction.scala

@@ -43,6 +43,8 @@ object DBFunction {
 
   case class Concat(exprs: NonEmptyList[SelectExpr]) extends DBFunction
 
+  case class Raw(name: String, exprs: NonEmptyList[SelectExpr]) extends DBFunction
+
   sealed trait Operator
   object Operator {
     case object Plus extends Operator

modules/store/src/main/scala/docspell/store/qb/DSL.scala

@@ -122,6 +122,9 @@ trait DSL extends DoobieMeta {
   def concat(expr: SelectExpr, exprs: SelectExpr*): DBFunction =
     DBFunction.Concat(Nel.of(expr, exprs: _*))
 
+  def rawFunction(name: String, expr: SelectExpr, more: SelectExpr*): DBFunction =
+    DBFunction.Raw(name, Nel.of(expr, more: _*))
+
   def const[A](value: A)(implicit P: Put[A]): SelectExpr.SelectConstant[A] =
     SelectExpr.SelectConstant(value, None)
 

modules/store/src/main/scala/docspell/store/qb/impl/DBFunctionBuilder.scala

@@ -61,6 +61,11 @@ object DBFunctionBuilder extends CommonBuilder {
 
       case DBFunction.Sum(expr) =>
         sql"SUM(" ++ SelectExprBuilder.build(expr) ++ fr")"
+
+      case DBFunction.Raw(name, exprs) =>
+        val n = Fragment.const0(name)
+        val inner = exprs.map(SelectExprBuilder.build).toList.reduce(_ ++ comma ++ _)
+        sql"$n($inner)"
     }
 
   def buildOperator(op: DBFunction.Operator): Fragment =

modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.queries
+
+import docspell.store.impl.TempFtsTable
+import docspell.store.qb.DSL._
+import docspell.store.qb._
+import docspell.store.records.RItem
+
+trait FtsSupport {
+
+  implicit final class SelectOps(select: Select) {
+    def joinFtsIdOnly(
+        itemTable: RItem.Table,
+        ftsTable: Option[TempFtsTable.Table]
+    ): Select =
+      ftsTable match {
+        case Some(ftst) =>
+          val tt = cteTable(ftst)
+          select
+            .appendCte(ftst.distinctCteSimple(tt.tableName))
+            .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+        case None =>
+          select
+      }
+
+    def joinFtsDetails(
+        itemTable: RItem.Table,
+        ftsTable: Option[TempFtsTable.Table]
+    ): Select =
+      ftsTable match {
+        case Some(ftst) =>
+          val tt = cteTable(ftst)
+          select
+            .appendCte(ftst.distinctCte(tt.tableName))
+            .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+        case None =>
+          select
+      }
+  }
+
+  def cteTable(ftsTable: TempFtsTable.Table) =
+    ftsTable.copy(tableName = "cte_fts")
+}
+
+object FtsSupport extends FtsSupport

modules/store/src/main/scala/docspell/store/queries/ListItem.scala

@@ -7,6 +7,7 @@
 package docspell.store.queries
 
 import docspell.common._
+import docspell.store.impl.TempFtsTable.ContextEntry
 
 case class ListItem(
     id: Ident,
@@ -22,5 +23,21 @@ case class ListItem(
     concPerson: Option[IdRef],
     concEquip: Option[IdRef],
     folder: Option[IdRef],
-    notes: Option[String]
+    notes: Option[String],
-)
+    context: Option[String]
+) {
+
+  def decodeContext: Option[Either[String, List[ContextEntry]]] =
+    context.map(_.trim).filter(_.nonEmpty).map { str =>
+      // This is a bit… well. The common denominator for the dbms used is string aggregation
+      // when combining multiple matches. So the `ContextEntry` objects are concatenated and
+      // separated by comma. TemplateFtsTable ensures than the single entries are all json
+      // objects.
+      val jsonStr = s"[ $str ]"
+      io.circe.parser
+        .decode[List[Option[ContextEntry]]](jsonStr)
+        .left
+        .map(_.getMessage)
+        .map(_.flatten)
+    }
+}

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -18,15 +18,17 @@ import docspell.common.{FileKey, IdRef, _}
 import docspell.query.ItemQuery.Expr.ValidItemStates
 import docspell.query.{ItemQuery, ItemQueryDsl}
 import docspell.store.Store
+import docspell.store.impl.TempFtsTable
 import docspell.store.qb.DSL._
 import docspell.store.qb._
 import docspell.store.qb.generator.{ItemQueryGenerator, Tables}
+import docspell.store.queries.Query.OrderSelect
 import docspell.store.records._
 
 import doobie.implicits._
 import doobie.{Query => _, _}
 
-object QItem {
+object QItem extends FtsSupport {
   private[this] val logger = docspell.logging.getLogger[ConnectionIO]
 
   private val equip = REquipment.as("e")
@@ -44,6 +46,35 @@ object QItem {
   private val ti = RTagItem.as("ti")
   private val meta = RFileMeta.as("fmeta")
 
+  private def orderSelect(ftsOpt: Option[TempFtsTable.Table]): OrderSelect =
+    new OrderSelect {
+      val item = i
+      val fts = ftsOpt
+    }
+
+  private val emptyString: SelectExpr = const("")
+
+  def queryItems(
+      q: Query,
+      today: LocalDate,
+      maxNoteLen: Int,
+      batch: Batch,
+      ftsTable: Option[TempFtsTable.Table]
+  ) = {
+    val cteFts = ftsTable.map(cteTable)
+    val sql =
+      findItemsBase(q.fix, today, maxNoteLen, cteFts)
+        .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
+        .joinFtsDetails(i, ftsTable)
+        .limit(batch)
+        .build
+
+    logger.stream.debug(s"List $batch items: $sql").drain ++
+      sql.query[ListItem].stream
+  }
+
+  // ----
+
   def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] =
     Select(count(a.id).s, from(a), a.itemId.in(items)).build
       .query[Int]
@@ -115,7 +146,12 @@ object QItem {
             ItemQuery.Expr.and(ValidItemStates, ItemQueryDsl.Q.itemIdsIn(nel.map(_.id)))
           val account = AccountId(collective, Ident.unsafe(""))
 
-          findItemsBase(Query.Fix(account, Some(expr), None), LocalDate.EPOCH, 0).build
+          findItemsBase(
+            Query.Fix(account, Some(expr), None),
+            LocalDate.EPOCH,
+            0,
+            None
+          ).build
             .query[ListItem]
             .to[Vector]
       }
@@ -130,7 +166,12 @@ object QItem {
       cv.itemId === itemId
     ).build.query[ItemFieldValue].to[Vector]
 
-  private def findItemsBase(q: Query.Fix, today: LocalDate, noteMaxLen: Int): Select = {
+  private def findItemsBase(
+      q: Query.Fix,
+      today: LocalDate,
+      noteMaxLen: Int,
+      ftsTable: Option[TempFtsTable.Table]
+  ): Select.Ordered = {
     val coll = q.account.collective
 
     Select(
@@ -154,8 +195,9 @@ object QItem {
         f.id.s,
         f.name.s,
         substring(i.notes.s, 1, noteMaxLen).s,
-        q.orderAsc
+        ftsTable.map(_.context.s).getOrElse(emptyString),
-          .map(of => coalesce(of(i).s, i.created.s).s)
+        q.order
+          .map(f => f(orderSelect(ftsTable)).expr)
           .getOrElse(i.created.s)
       ),
       from(i)
@@ -172,8 +214,8 @@ object QItem {
           )
       )
     ).orderBy(
-      q.orderAsc
+      q.order
-        .map(of => OrderBy.asc(coalesce(of(i).s, i.created.s).s))
+        .map(of => of(orderSelect(ftsTable)))
         .getOrElse(OrderBy.desc(coalesce(i.itemDate.s, i.created.s).s))
     )
   }
@@ -184,7 +226,7 @@ object QItem {
       today: LocalDate,
       maxFiles: Int
   ): Select =
-    findItemsBase(q.fix, today, 0)
+    findItemsBase(q.fix, today, 0, None)
       .changeFrom(_.innerJoin(a, a.itemId === i.id).innerJoin(as, a.id === as.id))
       .changeFrom(from =>
         ftype match {
@@ -277,26 +319,22 @@ object QItem {
       today: LocalDate,
       maxNoteLen: Int,
       batch: Batch
-  ): Stream[ConnectionIO, ListItem] = {
+  ): Stream[ConnectionIO, ListItem] =
-    val sql = findItemsBase(q.fix, today, maxNoteLen)
+    queryItems(q, today, maxNoteLen, batch, None)
-      .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
-      .limit(batch)
-      .build
-    logger.stream.trace(s"List $batch items: $sql").drain ++
-      sql.query[ListItem].stream
-  }
 
-  def searchStats(today: LocalDate)(q: Query): ConnectionIO[SearchSummary] =
+  def searchStats(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
+      q: Query
+  ): ConnectionIO[SearchSummary] =
     for {
-      count <- searchCountSummary(today)(q)
+      count <- searchCountSummary(today, ftsTable)(q)
-      tags <- searchTagSummary(today)(q)
+      tags <- searchTagSummary(today, ftsTable)(q)
-      cats <- searchTagCategorySummary(today)(q)
+      cats <- searchTagCategorySummary(today, ftsTable)(q)
-      fields <- searchFieldSummary(today)(q)
+      fields <- searchFieldSummary(today, ftsTable)(q)
-      folders <- searchFolderSummary(today)(q)
+      folders <- searchFolderSummary(today, ftsTable)(q)
-      orgs <- searchCorrOrgSummary(today)(q)
+      orgs <- searchCorrOrgSummary(today, ftsTable)(q)
-      corrPers <- searchCorrPersonSummary(today)(q)
+      corrPers <- searchCorrPersonSummary(today, ftsTable)(q)
-      concPers <- searchConcPersonSummary(today)(q)
+      concPers <- searchConcPersonSummary(today, ftsTable)(q)
-      concEquip <- searchConcEquipSummary(today)(q)
+      concEquip <- searchConcEquipSummary(today, ftsTable)(q)
     } yield SearchSummary(
       count,
       tags,
@@ -310,7 +348,8 @@ object QItem {
     )
 
   def searchTagCategorySummary(
-      today: LocalDate
+      today: LocalDate,
+      ftsTable: Option[TempFtsTable.Table]
   )(q: Query): ConnectionIO[List[CategoryCount]] = {
     val tagFrom =
       from(ti)
@@ -318,7 +357,8 @@ object QItem {
         .innerJoin(i, i.id === ti.itemId)
 
     val catCloud =
-      findItemsBase(q.fix, today, 0).unwrap
+      findItemsBase(q.fix, today, 0, None).unwrap
+        .joinFtsIdOnly(i, ftsTable)
         .withSelect(select(tag.category).append(countDistinct(i.id).as("num")))
         .changeFrom(_.prepend(tagFrom))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
@@ -334,14 +374,17 @@ object QItem {
     } yield existing ++ other.map(n => CategoryCount(n.some, 0))
   }
 
-  def searchTagSummary(today: LocalDate)(q: Query): ConnectionIO[List[TagCount]] = {
+  def searchTagSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
+      q: Query
+  ): ConnectionIO[List[TagCount]] = {
     val tagFrom =
       from(ti)
         .innerJoin(tag, tag.tid === ti.tagId)
         .innerJoin(i, i.id === ti.itemId)
 
     val tagCloud =
-      findItemsBase(q.fix, today, 0).unwrap
+      findItemsBase(q.fix, today, 0, None).unwrap
+        .joinFtsIdOnly(i, ftsTable)
         .withSelect(select(tag.all).append(countDistinct(i.id).as("num")))
         .changeFrom(_.prepend(tagFrom))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
@@ -358,39 +401,46 @@ object QItem {
     } yield existing ++ other.map(TagCount(_, 0))
   }
 
-  def searchCountSummary(today: LocalDate)(q: Query): ConnectionIO[Int] =
+  def searchCountSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
-    findItemsBase(q.fix, today, 0).unwrap
+      q: Query
+  ): ConnectionIO[Int] =
+    findItemsBase(q.fix, today, 0, None).unwrap
+      .joinFtsIdOnly(i, ftsTable)
       .withSelect(Nel.of(count(i.id).as("num")))
       .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
       .build
       .query[Int]
       .unique
 
-  def searchCorrOrgSummary(today: LocalDate)(q: Query): ConnectionIO[List[IdRefCount]] =
+  def searchCorrOrgSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
-    searchIdRefSummary(org.oid, org.name, i.corrOrg, today)(q)
-
-  def searchCorrPersonSummary(today: LocalDate)(
       q: Query
   ): ConnectionIO[List[IdRefCount]] =
-    searchIdRefSummary(pers0.pid, pers0.name, i.corrPerson, today)(q)
+    searchIdRefSummary(org.oid, org.name, i.corrOrg, today, ftsTable)(q)
 
-  def searchConcPersonSummary(today: LocalDate)(
+  def searchCorrPersonSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
       q: Query
   ): ConnectionIO[List[IdRefCount]] =
-    searchIdRefSummary(pers1.pid, pers1.name, i.concPerson, today)(q)
+    searchIdRefSummary(pers0.pid, pers0.name, i.corrPerson, today, ftsTable)(q)
 
-  def searchConcEquipSummary(today: LocalDate)(
+  def searchConcPersonSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
       q: Query
   ): ConnectionIO[List[IdRefCount]] =
-    searchIdRefSummary(equip.eid, equip.name, i.concEquipment, today)(q)
+    searchIdRefSummary(pers1.pid, pers1.name, i.concPerson, today, ftsTable)(q)
+
+  def searchConcEquipSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
+      q: Query
+  ): ConnectionIO[List[IdRefCount]] =
+    searchIdRefSummary(equip.eid, equip.name, i.concEquipment, today, ftsTable)(q)
 
   private def searchIdRefSummary(
       idCol: Column[Ident],
       nameCol: Column[String],
       fkCol: Column[Ident],
-      today: LocalDate
+      today: LocalDate,
+      ftsTable: Option[TempFtsTable.Table]
   )(q: Query): ConnectionIO[List[IdRefCount]] =
-    findItemsBase(q.fix, today, 0).unwrap
+    findItemsBase(q.fix, today, 0, None).unwrap
+      .joinFtsIdOnly(i, ftsTable)
       .withSelect(select(idCol, nameCol).append(count(idCol).as("num")))
       .changeWhere(c =>
         c && fkCol.isNotNull && queryCondition(today, q.fix.account.collective, q.cond)
@@ -400,9 +450,12 @@ object QItem {
       .query[IdRefCount]
       .to[List]
 
-  def searchFolderSummary(today: LocalDate)(q: Query): ConnectionIO[List[FolderCount]] = {
+  def searchFolderSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
+      q: Query
+  ): ConnectionIO[List[FolderCount]] = {
     val fu = RUser.as("fu")
-    findItemsBase(q.fix, today, 0).unwrap
+    findItemsBase(q.fix, today, 0, None).unwrap
+      .joinFtsIdOnly(i, ftsTable)
       .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num")))
       .changeFrom(_.innerJoin(fu, fu.uid === f.owner))
       .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
@@ -412,14 +465,17 @@ object QItem {
       .to[List]
   }
 
-  def searchFieldSummary(today: LocalDate)(q: Query): ConnectionIO[List[FieldStats]] = {
+  def searchFieldSummary(today: LocalDate, ftsTable: Option[TempFtsTable.Table])(
+      q: Query
+  ): ConnectionIO[List[FieldStats]] = {
     val fieldJoin =
       from(cv)
         .innerJoin(cf, cf.id === cv.field)
         .innerJoin(i, i.id === cv.itemId)
 
     val base =
-      findItemsBase(q.fix, today, 0).unwrap
+      findItemsBase(q.fix, today, 0, None).unwrap
+        .joinFtsIdOnly(i, ftsTable)
         .changeFrom(_.prepend(fieldJoin))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
         .groupBy(GroupBy(cf.all))
@@ -498,7 +554,7 @@ object QItem {
           )
         )
 
-      val from = findItemsBase(q.fix, today, maxNoteLen)
+      val from = findItemsBase(q.fix, today, maxNoteLen, None)
         .appendCte(cte)
         .appendSelect(Tids.weight.s)
         .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id))

modules/store/src/main/scala/docspell/store/queries/Query.scala

@@ -8,7 +8,9 @@ package docspell.store.queries
 
 import docspell.common._
 import docspell.query.ItemQuery
-import docspell.store.qb.Column
+import docspell.store.impl.TempFtsTable
+import docspell.store.qb.DSL._
+import docspell.store.qb.{Column, OrderBy}
 import docspell.store.records.RItem
 
 case class Query(fix: Query.Fix, cond: Query.QueryCond) {
@@ -16,7 +18,7 @@ case class Query(fix: Query.Fix, cond: Query.QueryCond) {
     copy(cond = f(cond))
 
   def withOrder(orderAsc: RItem.Table => Column[_]): Query =
-    withFix(_.copy(orderAsc = Some(orderAsc)))
+    withFix(_.copy(order = Some(_.byItemColumnAsc(orderAsc))))
 
   def withFix(f: Query.Fix => Query.Fix): Query =
     copy(fix = f(fix))
@@ -29,6 +31,19 @@ case class Query(fix: Query.Fix, cond: Query.QueryCond) {
 }
 
 object Query {
+  trait OrderSelect {
+    def item: RItem.Table
+    def fts: Option[TempFtsTable.Table]
+
+    def byDefault: OrderBy =
+      OrderBy.desc(coalesce(item.itemDate.s, item.created.s).s)
+
+    def byItemColumnAsc(f: RItem.Table => Column[_]): OrderBy =
+      OrderBy.asc(coalesce(f(item).s, item.created.s).s)
+
+    def byScore: OrderBy =
+      fts.map(t => OrderBy.desc(t.score.s)).getOrElse(byDefault)
+  }
 
   def apply(fix: Fix): Query =
     Query(fix, QueryExpr(None))
@@ -36,7 +51,7 @@ object Query {
   case class Fix(
       account: AccountId,
       query: Option[ItemQuery.Expr],
-      orderAsc: Option[RItem.Table => Column[_]]
+      order: Option[OrderSelect => OrderBy]
   ) {
 
     def isEmpty: Boolean =

modules/store/src/test/scala/docspell/store/DatabaseTest.scala

@@ -1,19 +1,27 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
 package docspell.store
 
+import java.util.UUID
+
 import cats.effect._
+
+import docspell.common._
+import docspell.logging.TestLoggingConfig
+
+import com.dimafeng.testcontainers.munit.fixtures.TestContainersFixtures
 import com.dimafeng.testcontainers.{
   JdbcDatabaseContainer,
   MariaDBContainer,
   PostgreSQLContainer
 }
-import com.dimafeng.testcontainers.munit.fixtures.TestContainersFixtures
+import doobie._
-import docspell.common._
-import docspell.logging.TestLoggingConfig
 import munit.CatsEffectSuite
 import org.testcontainers.utility.DockerImageName
-import doobie._
-
-import java.util.UUID
 
 trait DatabaseTest
     extends CatsEffectSuite

modules/store/src/test/scala/docspell/store/Docker.scala

@@ -1,3 +1,9 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
 package docspell.store
 
 import cats.effect._

modules/store/src/test/scala/docspell/store/impl/TempFtsTableTest.scala

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.impl
+
+import java.time.LocalDate
+
+import cats.effect.IO
+import cats.syntax.option._
+import cats.syntax.traverse._
+import fs2.Stream
+
+import docspell.common._
+import docspell.ftsclient.FtsResult
+import docspell.ftsclient.FtsResult.{AttachmentData, ItemMatch}
+import docspell.logging.Level
+import docspell.store._
+import docspell.store.impl.TempFtsTable.Row
+import docspell.store.qb.DSL._
+import docspell.store.qb._
+import docspell.store.queries.{QItem, Query}
+import docspell.store.records.{RCollective, RItem}
+
+import doobie._
+
+class TempFtsTableTest extends DatabaseTest {
+  private[this] val logger = docspell.logging.getLogger[IO]
+  override def rootMinimumLevel = Level.Info
+
+  override def munitFixtures = postgresAll ++ mariaDbAll ++ h2All
+
+  def id(str: String): Ident = Ident.unsafe(str)
+
+  def stores: (Store[IO], Store[IO], Store[IO]) =
+    (pgStore(), mariaStore(), h2Store())
+
+  test("create temporary table") {
+    val (pg, maria, h2) = stores
+    for {
+      _ <- assertCreateTempTable(pg)
+      _ <- assertCreateTempTable(maria)
+      _ <- assertCreateTempTable(h2)
+    } yield ()
+  }
+
+  test("query items sql") {
+    val (pg, maria, h2) = stores
+    for {
+      _ <- prepareItems(pg)
+      _ <- prepareItems(maria)
+      _ <- prepareItems(h2)
+      _ <- assertQueryItem(pg, ftsResults(10, 10))
+      _ <- assertQueryItem(pg, ftsResults(3000, 500))
+      _ <- assertQueryItem(pg, ftsResults(3000, 500))
+      _ <- assertQueryItem(maria, ftsResults(10, 10))
+      _ <- assertQueryItem(maria, ftsResults(3000, 500))
+      _ <- assertQueryItem(h2, ftsResults(10, 10))
+      _ <- assertQueryItem(h2, ftsResults(3000, 500))
+    } yield ()
+  }
+
+  def prepareItems(store: Store[IO]) =
+    for {
+      _ <- store.transact(RCollective.insert(makeCollective(DocspellSystem.user)))
+      items = (0 until 200)
+        .map(makeItem(_, DocspellSystem.user))
+        .toList
+      _ <- items.traverse(i => store.transact(RItem.insert(i)))
+    } yield ()
+
+  def assertCreateTempTable(store: Store[IO]) = {
+    val insertRows =
+      List(
+        Row(id("abc-def"), None, None),
+        Row(id("abc-123"), Some(1.56), None),
+        Row(id("zyx-321"), None, None)
+      )
+    val create =
+      for {
+        table <- TempFtsTable.createTable(store.dbms, "tt")
+        n <- table.insertAll(insertRows)
+        _ <- table.createIndex
+        rows <- Select(select(table.all), from(table))
+          .orderBy(table.id)
+          .build
+          .query[Row]
+          .to[List]
+      } yield (n, rows)
+
+    val verify =
+      store.transact(create).map { case (inserted, rows) =>
+        if (store.dbms != Db.MariaDB) {
+          assertEquals(inserted, 3)
+        }
+        assertEquals(rows, insertRows.sortBy(_.id))
+      }
+
+    verify *> verify
+  }
+
+  def assertQueryItem(store: Store[IO], ftsResults: Stream[ConnectionIO, FtsResult]) =
+    for {
+      today <- IO(LocalDate.now())
+      account = DocspellSystem.account
+      tempTable = ftsResults
+        .through(TempFtsTable.prepareTable(store.dbms, "fts_result"))
+        .compile
+        .lastOrError
+      q = Query(Query.Fix(account, None, None), Query.QueryExpr(None))
+      timed <- Duration.stopTime[IO]
+      items <- store
+        .transact(
+          tempTable.flatMap(t =>
+            QItem
+              .queryItems(q, today, 0, Batch.limit(10), t.some)
+              .compile
+              .to(List)
+          )
+        )
+      duration <- timed
+      _ <- logger.info(s"Join took: ${duration.formatExact}")
+
+    } yield {
+      assert(items.nonEmpty)
+      assert(items.head.context.isDefined)
+    }
+
+  def ftsResult(start: Int, end: Int): FtsResult = {
+    def matchData(n: Int): List[ItemMatch] =
+      List(
+        ItemMatch(
+          id(s"m$n"),
+          id(s"item-$n"),
+          DocspellSystem.user,
+          math.random(),
+          FtsResult.ItemData
+        ),
+        ItemMatch(
+          id(s"m$n-1"),
+          id(s"item-$n"),
+          DocspellSystem.user,
+          math.random(),
+          AttachmentData(id(s"item-$n-attach-1"), "attachment.pdf")
+        )
+      )
+
+    val hl =
+      (start until end)
+        .flatMap(n =>
+          List(
+            id(s"m$n-1") -> List("this *a test* please"),
+            id(s"m$n") -> List("only **items** here")
+          )
+        )
+        .toMap
+
+    FtsResult.empty
+      .copy(
+        count = end,
+        highlight = hl,
+        results = (start until end).toList.flatMap(matchData)
+      )
+  }
+
+  def ftsResults(len: Int, chunkSize: Int): Stream[ConnectionIO, FtsResult] = {
+    val chunks = len / chunkSize
+    Stream.range(0, chunks).map { n =>
+      val start = n * chunkSize
+      val end = start + chunkSize
+      ftsResult(start, end)
+    }
+  }
+
+  def makeCollective(cid: Ident): RCollective =
+    RCollective(cid, CollectiveState.Active, Language.English, true, Timestamp.Epoch)
+
+  def makeItem(n: Int, cid: Ident): RItem =
+    RItem(
+      id(s"item-$n"),
+      cid,
+      s"item $n",
+      None,
+      "test",
+      Direction.Incoming,
+      ItemState.Created,
+      None,
+      None,
+      None,
+      None,
+      None,
+      None,
+      Timestamp.Epoch,
+      Timestamp.Epoch,
+      None,
+      None
+    )
+}

modules/store/src/test/scala/docspell/store/impl/TempIdTableTest.scala

@@ -1,55 +0,0 @@
-package docspell.store.impl
-
-import cats.effect.IO
-import docspell.common.Ident
-import docspell.store._
-import docspell.store.impl.TempIdTable.Row
-import docspell.store.qb._
-import docspell.store.qb.DSL._
-
-class TempIdTableTest extends DatabaseTest {
-
-  override def munitFixtures = postgresAll ++ mariaDbAll ++ h2All
-
-  def id(str: String): Ident = Ident.unsafe(str)
-
-  test("create temporary table postgres") {
-    val store = pgStore()
-    assertCreateTempTable(store)
-  }
-
-  test("create temporary table mariadb") {
-    val store = mariaStore()
-    assertCreateTempTable(store)
-  }
-
-  test("create temporary table h2") {
-    val store = h2Store()
-    assertCreateTempTable(store)
-  }
-
-  def assertCreateTempTable(store: Store[IO]) = {
-    val insertRows = List(Row(id("abc-def")), Row(id("abc-123")), Row(id("zyx-321")))
-    val create =
-      for {
-        table <- TempIdTable.createTable(store.dbms, "tt")
-        n <- table.insertAll(insertRows)
-        _ <- table.createIndex
-        rows <- Select(select(table.all), from(table))
-          .orderBy(table.id)
-          .build
-          .query[Row]
-          .to[List]
-      } yield (n, rows)
-
-    val verify =
-      store.transact(create).map { case (inserted, rows) =>
-        if (store.dbms != Db.MariaDB) {
-          assertEquals(inserted, 3)
-        }
-        assertEquals(rows, insertRows.sortBy(_.id))
-      }
-
-    verify *> verify
-  }
-}

modules/store/src/test/scala/docspell/store/migrate/MigrateTest.scala

@@ -7,7 +7,9 @@
 package docspell.store.migrate
 
 import cats.effect._
+
 import docspell.store.{DatabaseTest, SchemaMigrateConfig, StoreFixture}
+
 import org.flywaydb.core.api.output.MigrateResult
 
 class MigrateTest extends DatabaseTest {