Fixup for deleting items

First, when checking for existence of a file, deleted items are not
conisdered.

The working with fulltext search has been changed: deleted items are
removed from fulltext index and are re-added when they are restored.
The fulltext index currently doesn't hold the item state and it would
mean much more work to introduce it into the index (or, worse, to
reprocess the results from the index). Thus, deleted items can only be
searched using database queries. It is probably a very rare use case
when fulltext search should be applied to deleted items. They have
been deleted for a reason and the most likely case is that they are
simply removed.

Refs: #347

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -13,6 +13,7 @@ import cats.implicits._
 import fs2.concurrent.SignallingRef
 
 import docspell.analysis.TextAnalyser
+import docspell.backend.fulltext.CreateIndex
 import docspell.backend.ops._
 import docspell.common._
 import docspell.ftsclient.FtsClient
@@ -116,7 +117,8 @@ object JoexAppImpl {
       joex          <- OJoex(client, store)
       upload        <- OUpload(store, queue, cfg.files, joex)
       fts           <- createFtsClient(cfg)(httpClient)
-      itemOps       <- OItem(store, fts, queue, joex)
+      createIndex   <- CreateIndex.resource(fts, store)
+      itemOps       <- OItem(store, fts, createIndex, queue, joex)
       itemSearchOps <- OItemSearch(store)
       analyser      <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig)
       regexNer      <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store)
@@ -155,14 +157,14 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             MigrationTask.taskName,
-            MigrationTask[F](cfg.fullTextSearch, fts),
+            MigrationTask[F](cfg.fullTextSearch, fts, createIndex),
             MigrationTask.onCancel[F]
           )
         )
         .withTask(
           JobTask.json(
             ReIndexTask.taskName,
-            ReIndexTask[F](cfg.fullTextSearch, fts),
+            ReIndexTask[F](cfg.fullTextSearch, fts, createIndex),
             ReIndexTask.onCancel[F]
           )
         )

modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala

@@ -6,6 +6,7 @@
 
 package docspell.joex.fts
 
+import docspell.backend.fulltext.CreateIndex
 import docspell.common.Logger
 import docspell.ftsclient.FtsClient
 import docspell.joex.Config
@@ -15,6 +16,7 @@ import docspell.store.Store
 case class FtsContext[F[_]](
     cfg: Config.FullTextSearch,
     store: Store[F],
+    fulltext: CreateIndex[F],
     fts: FtsClient[F],
     logger: Logger[F]
 )
@@ -24,7 +26,8 @@ object FtsContext {
   def apply[F[_]](
       cfg: Config.FullTextSearch,
       fts: FtsClient[F],
+      fulltext: CreateIndex[F],
       ctx: Context[F, _]
   ): FtsContext[F] =
-    FtsContext(cfg, ctx.store, fts, ctx.logger)
+    FtsContext(cfg, ctx.store, fulltext, fts, ctx.logger)
 }

modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala

@@ -10,11 +10,11 @@ import cats._
 import cats.data.{Kleisli, NonEmptyList}
 import cats.implicits._
 
+import docspell.backend.fulltext.CreateIndex
 import docspell.common._
 import docspell.ftsclient._
 import docspell.joex.Config
 import docspell.joex.scheduler.Context
-import docspell.store.queries.{QAttachment, QItem}
 
 object FtsWork {
   import syntax._
@@ -88,36 +88,8 @@ object FtsWork {
     log[F](_.info("Inserting all data to index")) ++ FtsWork
       .all(
         FtsWork(ctx =>
-          ctx.fts.indexData(
-            ctx.logger,
-            ctx.store
-              .transact(
-                QAttachment
-                  .allAttachmentMetaAndName(coll, ctx.cfg.migration.indexAllChunk)
-              )
-              .map(caa =>
-                TextData
-                  .attachment(
-                    caa.item,
-                    caa.id,
-                    caa.collective,
-                    caa.folder,
-                    caa.lang,
-                    caa.name,
-                    caa.content
-                  )
-              )
-          )
-        ),
-        FtsWork(ctx =>
-          ctx.fts.indexData(
-            ctx.logger,
-            ctx.store
-              .transact(QItem.allNameAndNotes(coll, ctx.cfg.migration.indexAllChunk * 5))
-              .map(nn =>
-                TextData.item(nn.id, nn.collective, nn.folder, Option(nn.name), nn.notes)
-              )
-          )
+          ctx.fulltext
+            .reIndexData(ctx.logger, coll, None, ctx.cfg.migration.indexAllChunk)
         )
       )
 
@@ -133,9 +105,10 @@ object FtsWork {
 
       def forContext(
           cfg: Config.FullTextSearch,
-          fts: FtsClient[F]
+          fts: FtsClient[F],
+          fulltext: CreateIndex[F]
       ): Kleisli[F, Context[F, _], Unit] =
-        mt.local(ctx => FtsContext(cfg, fts, ctx))
+        mt.local(ctx => FtsContext(cfg, fts, fulltext, ctx))
     }
   }
 }

modules/joex/src/main/scala/docspell/joex/fts/Migration.scala

@@ -11,6 +11,7 @@ import cats.effect._
 import cats.implicits._
 import cats.{Applicative, FlatMap, Traverse}
 
+import docspell.backend.fulltext.CreateIndex
 import docspell.common._
 import docspell.ftsclient._
 import docspell.joex.Config
@@ -38,9 +39,10 @@ object Migration {
       cfg: Config.FullTextSearch,
       fts: FtsClient[F],
       store: Store[F],
+      createIndex: CreateIndex[F],
       logger: Logger[F]
   ): Kleisli[F, List[Migration[F]], Unit] = {
-    val ctx = FtsContext(cfg, store, fts, logger)
+    val ctx = FtsContext(cfg, store, createIndex, fts, logger)
     Kleisli { migs =>
       if (migs.isEmpty) logger.info("No fulltext search migrations to run.")
       else Traverse[List].sequence(migs.map(applySingle[F](ctx))).map(_ => ())

modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala

@@ -9,6 +9,7 @@ package docspell.joex.fts
 import cats.effect._
 import cats.implicits._
 
+import docspell.backend.fulltext.CreateIndex
 import docspell.common._
 import docspell.ftsclient._
 import docspell.joex.Config
@@ -20,7 +21,8 @@ object MigrationTask {
 
   def apply[F[_]: Async](
       cfg: Config.FullTextSearch,
-      fts: FtsClient[F]
+      fts: FtsClient[F],
+      createIndex: CreateIndex[F]
   ): Task[F, Unit, Unit] =
     Task
       .log[F, Unit](_.info(s"Running full-text-index migrations now"))
@@ -28,7 +30,7 @@ object MigrationTask {
         Task(ctx =>
           for {
             migs <- migrationTasks[F](fts)
-            res  <- Migration[F](cfg, fts, ctx.store, ctx.logger).run(migs)
+            res  <- Migration[F](cfg, fts, ctx.store, createIndex, ctx.logger).run(migs)
           } yield res
         )
       )

modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala

@@ -8,6 +8,7 @@ package docspell.joex.fts
 
 import cats.effect._
 
+import docspell.backend.fulltext.CreateIndex
 import docspell.common._
 import docspell.ftsclient._
 import docspell.joex.Config
@@ -22,12 +23,15 @@ object ReIndexTask {
 
   def apply[F[_]: Async](
       cfg: Config.FullTextSearch,
-      fts: FtsClient[F]
+      fts: FtsClient[F],
+      fulltext: CreateIndex[F]
   ): Task[F, Args, Unit] =
     Task
       .log[F, Args](_.info(s"Running full-text re-index now"))
       .flatMap(_ =>
-        Task(ctx => clearData[F](ctx.args.collective).forContext(cfg, fts).run(ctx))
+        Task(ctx =>
+          clearData[F](ctx.args.collective).forContext(cfg, fts, fulltext).run(ctx)
+        )
       )
 
   def onCancel[F[_]]: Task[F, Args, Unit] =