Introducing fts client into codebase

modules/backend/src/main/scala/docspell/backend/BackendApp.scala

@@ -1,12 +1,16 @@
 package docspell.backend
 
 import cats.effect.{Blocker, ConcurrentEffect, ContextShift, Resource}
+import org.http4s.client.blaze.BlazeClientBuilder
+
 import docspell.backend.auth.Login
 import docspell.backend.ops._
 import docspell.backend.signup.OSignup
+import docspell.joexapi.client.JoexClient
 import docspell.store.Store
 import docspell.store.queue.JobQueue
 import docspell.store.usertask.UserTaskStore
+import docspell.ftssolr.SolrFtsClient
 
 import scala.concurrent.ExecutionContext
 import emil.javamail.{JavaMailEmil, Settings}
@@ -25,6 +29,7 @@ trait BackendApp[F[_]] {
   def job: OJob[F]
   def item: OItem[F]
   def itemSearch: OItemSearch[F]
+  def fulltext: OFulltext[F]
   def mail: OMail[F]
   def joex: OJoex[F]
   def userTask: OUserTask[F]
@@ -39,6 +44,7 @@ object BackendApp {
       blocker: Blocker
   ): Resource[F, BackendApp[F]] =
     for {
+      httpClient     <- BlazeClientBuilder[F](httpClientEc).resource
       utStore        <- UserTaskStore(store)
       queue          <- JobQueue(store)
       loginImpl      <- Login[F](store)
@@ -48,12 +54,14 @@ object BackendApp {
       tagImpl        <- OTag[F](store)
       equipImpl      <- OEquipment[F](store)
       orgImpl        <- OOrganization(store)
-      joexImpl       <- OJoex.create(httpClientEc, store)
+      joexImpl       <- OJoex(JoexClient(httpClient), store)
       uploadImpl     <- OUpload(store, queue, cfg.files, joexImpl)
       nodeImpl       <- ONode(store)
       jobImpl        <- OJob(store, joexImpl)
       itemImpl       <- OItem(store)
       itemSearchImpl <- OItemSearch(store)
+      solrFts        <- SolrFtsClient(cfg.fullTextSearch.solr, httpClient)
+      fulltextImpl   <- OFulltext(itemSearchImpl, solrFts)
       javaEmil =
         JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))
       mailImpl     <- OMail(store, javaEmil)
@@ -71,6 +79,7 @@ object BackendApp {
       val job                        = jobImpl
       val item                       = itemImpl
       val itemSearch                 = itemSearchImpl
+      val fulltext                   = fulltextImpl
       val mail                       = mailImpl
       val joex                       = joexImpl
       val userTask                   = userTaskImpl

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -3,16 +3,19 @@ package docspell.backend
 import docspell.backend.signup.{Config => SignupConfig}
 import docspell.common._
 import docspell.store.JdbcConfig
+import docspell.ftssolr.SolrConfig
 
 case class Config(
     mailDebug: Boolean,
     jdbc: JdbcConfig,
     signup: SignupConfig,
-    files: Config.Files
+    files: Config.Files,
+    fullTextSearch: Config.FullTextSearch
 ) {}
 
 object Config {
 
   case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType])
 
+  case class FullTextSearch(enabled: Boolean, solr: SolrConfig)
 }

modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala

@@ -0,0 +1,76 @@
+package docspell.backend.ops
+
+import cats.effect._
+import cats.implicits._
+import fs2.Stream
+import docspell.ftsclient._
+import OItemSearch.{Batch, ListItem, ListItemWithTags, Query}
+
+trait OFulltext[F[_]] {
+
+  def findItems(q: Query, fts: String, batch: Batch): F[Vector[ListItem]]
+
+  /** Same as `findItems` but does more queries per item to find all tags. */
+  def findItemsWithTags(q: Query, fts: String, batch: Batch): F[Vector[ListItemWithTags]]
+
+}
+
+object OFulltext {
+  // maybe use a temporary table? could run fts and do .take(batch.limit) and store this in sql
+  // then run a query
+  // check if supported by mariadb, postgres and h2. seems like it is supported everywhere
+
+
+  def apply[F[_]: Effect](
+      itemSearch: OItemSearch[F],
+      fts: FtsClient[F]
+  ): Resource[F, OFulltext[F]] =
+    Resource.pure[F, OFulltext[F]](new OFulltext[F] {
+
+      def findItems(q: Query, ftsQ: String, batch: Batch): F[Vector[ListItem]] =
+        findItemsFts(q, ftsQ, batch, itemSearch.findItems)
+          .take(batch.limit.toLong)
+          .compile
+          .toVector
+
+      def findItemsWithTags(
+          q: Query,
+          ftsQ: String,
+          batch: Batch
+      ): F[Vector[ListItemWithTags]] =
+        findItemsFts(q, ftsQ, batch, itemSearch.findItemsWithTags)
+          .take(batch.limit.toLong)
+          .compile
+          .toVector
+
+
+      private def findItemsFts[A](
+          q: Query,
+          ftsQ: String,
+          batch: Batch,
+          search: (Query, Batch) => F[Vector[A]]
+      ): Stream[F, A] = {
+        val fq = FtsQuery(ftsQ, q.collective, batch.limit, batch.offset)
+
+        val qres =
+          for {
+            items <-
+              fts
+                .searchBasic(fq)
+                .map(_.item)
+                .compile
+                .toVector
+                .map(_.toSet)
+            sq = q.copy(itemIds = Some(items))
+            res <- search(sq, batch)
+          } yield res
+
+        Stream.eval(qres).flatMap { v =>
+          val results = Stream.emits(v)
+          if (v.size < batch.limit) results
+          else results ++ findItemsFts(q, ftsQ, batch.next, search)
+        }
+      }
+
+    })
+}