Fixup for deleting items

First, when checking for existence of a file, deleted items are not
conisdered.

The working with fulltext search has been changed: deleted items are
removed from fulltext index and are re-added when they are restored.
The fulltext index currently doesn't hold the item state and it would
mean much more work to introduce it into the index (or, worse, to
reprocess the results from the index). Thus, deleted items can only be
searched using database queries. It is probably a very rare use case
when fulltext search should be applied to deleted items. They have
been deleted for a reason and the most likely case is that they are
simply removed.

Refs: #347

modules/backend/src/main/scala/docspell/backend/BackendApp.scala

@@ -11,6 +11,7 @@ import scala.concurrent.ExecutionContext
 import cats.effect._
 
 import docspell.backend.auth.Login
+import docspell.backend.fulltext.CreateIndex
 import docspell.backend.ops._
 import docspell.backend.signup.OSignup
 import docspell.ftsclient.FtsClient
@@ -69,7 +70,8 @@ object BackendApp {
       uploadImpl     <- OUpload(store, queue, cfg.files, joexImpl)
       nodeImpl       <- ONode(store)
       jobImpl        <- OJob(store, joexImpl)
-      itemImpl       <- OItem(store, ftsClient, queue, joexImpl)
+      createIndex    <- CreateIndex.resource(ftsClient, store)
+      itemImpl       <- OItem(store, ftsClient, createIndex, queue, joexImpl)
       itemSearchImpl <- OItemSearch(store)
       fulltextImpl   <- OFulltext(itemSearchImpl, ftsClient, store, queue, joexImpl)
       javaEmil =

modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Docspell Contributors
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package docspell.backend.fulltext
+
+import cats.data.NonEmptyList
+import cats.effect._
+
+import docspell.common._
+import docspell.ftsclient.FtsClient
+import docspell.ftsclient.TextData
+import docspell.store.Store
+import docspell.store.queries.QAttachment
+import docspell.store.queries.QItem
+
+trait CreateIndex[F[_]] {
+
+  /** Low-level function to re-index data. It is not submitted as a job,
+    * but invoked on the current machine.
+    */
+  def reIndexData(
+      logger: Logger[F],
+      collective: Option[Ident],
+      itemIds: Option[NonEmptyList[Ident]],
+      chunkSize: Int
+  ): F[Unit]
+
+}
+
+object CreateIndex {
+
+  def resource[F[_]](fts: FtsClient[F], store: Store[F]): Resource[F, CreateIndex[F]] =
+    Resource.pure(apply(fts, store))
+
+  def apply[F[_]](fts: FtsClient[F], store: Store[F]): CreateIndex[F] =
+    new CreateIndex[F] {
+      def reIndexData(
+          logger: Logger[F],
+          collective: Option[Ident],
+          itemIds: Option[NonEmptyList[Ident]],
+          chunkSize: Int
+      ): F[Unit] = {
+        val attachs = store
+          .transact(QAttachment.allAttachmentMetaAndName(collective, itemIds, chunkSize))
+          .map(caa =>
+            TextData
+              .attachment(
+                caa.item,
+                caa.id,
+                caa.collective,
+                caa.folder,
+                caa.lang,
+                caa.name,
+                caa.content
+              )
+          )
+
+        val items = store
+          .transact(QItem.allNameAndNotes(collective, itemIds, chunkSize))
+          .map(nn =>
+            TextData.item(nn.id, nn.collective, nn.folder, Option(nn.name), nn.notes)
+          )
+
+        fts.indexData(logger, attachs ++ items)
+      }
+    }
+
+}

modules/backend/src/main/scala/docspell/backend/ops/OItem.scala

@@ -11,6 +11,7 @@ import cats.effect.{Async, Resource}
 import cats.implicits._
 
 import docspell.backend.JobFactory
+import docspell.backend.fulltext.CreateIndex
 import docspell.common._
 import docspell.ftsclient.FtsClient
 import docspell.store.queries.{QAttachment, QItem, QMoveAttachment}
@@ -212,6 +213,7 @@ object OItem {
   def apply[F[_]: Async](
       store: Store[F],
       fts: FtsClient[F],
+      createIndex: CreateIndex[F],
       queue: JobQueue[F],
       joex: OJoex[F]
   ): Resource[F, OItem[F]] =
@@ -588,12 +590,13 @@ object OItem {
             items: NonEmptyList[Ident],
             collective: Ident
         ): F[UpdateResult] =
-          UpdateResult.fromUpdate(
-            store
+          UpdateResult.fromUpdate(for {
+            n <- store
               .transact(
                 RItem.restoreStateForCollective(items, ItemState.Created, collective)
               )
-          )
+            _ <- createIndex.reIndexData(logger, collective.some, items.some, 10)
+          } yield n)
 
         def setItemDate(
             items: NonEmptyList[Ident],
@@ -628,7 +631,10 @@ object OItem {
           } yield n
 
         def setDeletedState(items: NonEmptyList[Ident], collective: Ident): F[Int] =
-          store.transact(RItem.setState(items, collective, ItemState.Deleted))
+          for {
+            n <- store.transact(RItem.setState(items, collective, ItemState.Deleted))
+            _ <- items.traverse(id => fts.removeItem(logger, id))
+          } yield n
 
         def getProposals(item: Ident, collective: Ident): F[MetaProposalList] =
           store.transact(QAttachment.getMetaProposals(item, collective))

modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala

@@ -231,13 +231,16 @@ object OSimpleSearch {
         case Some(ftq) if settings.useFTS =>
           if (q.isEmpty) {
             logger.debug(s"Using index only search: $fulltextQuery")
-            fts
-              .findIndexOnly(settings.maxNoteLen)(
-                OFulltext.FtsInput(ftq),
-                q.fix.account,
-                settings.batch
-              )
-              .map(Items.ftsItemsFull(true))
+            if (settings.searchMode == SearchMode.Trashed)
+              Items.ftsItemsFull(true)(Vector.empty).pure[F]
+            else
+              fts
+                .findIndexOnly(settings.maxNoteLen)(
+                  OFulltext.FtsInput(ftq),
+                  q.fix.account,
+                  settings.batch
+                )
+                .map(Items.ftsItemsFull(true))
           } else if (settings.resolveDetails) {
             logger.debug(
               s"Using index+sql search with tags: $validItemQuery / $fulltextQuery"