Refactor full-text migrations and add folder to solr schema

modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala

@@ -17,11 +17,12 @@ import org.log4s.getLogger
   */
 trait FtsClient[F[_]] {
 
-  /** Initialization tasks. This is called exactly once and then never
+  /** Initialization tasks. This is called exactly once at the very
+    * beginning when initializing the full-text index and then never
     * again (except when re-indexing everything). It may be used to
     * setup the database.
     */
-  def initialize: F[Unit]
+  def initialize: List[FtsMigration[F]]
 
   /** Run a full-text search. */
   def search(q: FtsQuery): F[FtsResult]
@@ -107,8 +108,8 @@ object FtsClient {
     new FtsClient[F] {
       private[this] val logger = Logger.log4s[F](getLogger)
 
-      def initialize: F[Unit] =
+      def initialize: List[FtsMigration[F]] =
-        logger.info("Full-text search is disabled!")
+        Nil
 
       def search(q: FtsQuery): F[FtsResult] =
         logger.warn("Full-text search is disabled!") *> FtsResult.empty.pure[F]

modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala

@@ -0,0 +1,24 @@
+package docspell.ftsclient
+
+import docspell.common._
+
+final case class FtsMigration[F[_]](
+    version: Int,
+    engine: Ident,
+    description: String,
+    task: F[FtsMigration.Result]
+)
+
+object FtsMigration {
+
+  sealed trait Result
+  object Result {
+    case object WorkDone   extends Result
+    case object ReIndexAll extends Result
+    case object IndexAll   extends Result
+
+    def workDone: Result   = WorkDone
+    def reIndexAll: Result = ReIndexAll
+    def indexAll: Result   = IndexAll
+  }
+}

modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala

@@ -25,6 +25,7 @@ object Field {
   val content_en     = Field("content_en")
   val itemName       = Field("itemName")
   val itemNotes      = Field("itemNotes")
+  val folderId       = Field("folder")
 
   def contentField(lang: Language): Field =
     lang match {

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala

@@ -17,7 +17,7 @@ final class SolrFtsClient[F[_]: Effect](
     solrQuery: SolrQuery[F]
 ) extends FtsClient[F] {
 
-  def initialize: F[Unit] =
+  def initialize: List[FtsMigration[F]] =
     solrSetup.setupSchema
 
   def search(q: FtsQuery): F[FtsResult] =

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala

@@ -4,6 +4,7 @@ import cats.effect._
 import cats.implicits._
 
 import docspell.common._
+import docspell.ftsclient.FtsMigration
 
 import _root_.io.circe._
 import _root_.io.circe.generic.semiauto._
@@ -15,21 +16,48 @@ import org.http4s.client.dsl.Http4sClientDsl
 
 trait SolrSetup[F[_]] {
 
-  def setupSchema: F[Unit]
+  def setupSchema: List[FtsMigration[F]]
 
 }
 
 object SolrSetup {
+  private val solrEngine = Ident.unsafe("solr")
 
   def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrSetup[F] = {
     val dsl = new Http4sClientDsl[F] {}
     import dsl._
 
     new SolrSetup[F] {
+
       val url = (Uri.unsafeFromString(cfg.url.asString) / "schema")
         .withQueryParam("commitWithin", cfg.commitWithin.toString)
 
-      def setupSchema: F[Unit] = {
+      def setupSchema: List[FtsMigration[F]] =
+        List(
+          FtsMigration[F](
+            1,
+            solrEngine,
+            "Initialize",
+            setupCoreSchema.map(_ => FtsMigration.Result.workDone)
+          ),
+          FtsMigration[F](
+            3,
+            solrEngine,
+            "Add folder field",
+            addFolderField.map(_ => FtsMigration.Result.workDone)
+          ),
+          FtsMigration[F](
+            4,
+            solrEngine,
+            "Index all from database",
+            FtsMigration.Result.indexAll.pure[F]
+          )
+        )
+
+      def addFolderField: F[Unit] =
+        addStringField(Field.folderId)
+
+      def setupCoreSchema: F[Unit] = {
         val cmds0 =
           List(
             Field.id,

modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala

@@ -1,9 +1,8 @@
 package docspell.joex.fts
 
 import cats.data.{Kleisli, NonEmptyList}
-import cats.effect._
 import cats.implicits._
-import cats.{ApplicativeError, FlatMap, Semigroup}
+import cats.{Applicative, ApplicativeError, FlatMap, Monad, Semigroup}
 
 import docspell.common._
 import docspell.ftsclient._
@@ -15,6 +14,19 @@ object FtsWork {
   def apply[F[_]](f: FtsContext[F] => F[Unit]): FtsWork[F] =
     Kleisli(f)
 
+  def allInitializeTasks[F[_]: Monad]: FtsWork[F] =
+    FtsWork[F](_ => ().pure[F]).tap[FtsContext[F]].flatMap { ctx =>
+      NonEmptyList.fromList(ctx.fts.initialize.map(fm => from[F](fm.task))) match {
+        case Some(nel) =>
+          nel.reduce(semigroup[F])
+        case None =>
+          FtsWork[F](_ => ().pure[F])
+      }
+    }
+
+  def from[F[_]: FlatMap: Applicative](t: F[FtsMigration.Result]): FtsWork[F] =
+    Kleisli.liftF(t).flatMap(transformResult[F])
+
   def all[F[_]: FlatMap](
       m0: FtsWork[F],
       mn: FtsWork[F]*
@@ -24,14 +36,25 @@ object FtsWork {
   implicit def semigroup[F[_]: FlatMap]: Semigroup[FtsWork[F]] =
     Semigroup.instance((mt1, mt2) => mt1.flatMap(_ => mt2))
 
+  private def transformResult[F[_]: Applicative: FlatMap](
+      r: FtsMigration.Result
+  ): FtsWork[F] =
+    r match {
+      case FtsMigration.Result.WorkDone =>
+        Kleisli.pure(())
+
+      case FtsMigration.Result.IndexAll =>
+        insertAll[F](None)
+
+      case FtsMigration.Result.ReIndexAll =>
+        clearIndex[F](None) >> insertAll[F](None)
+    }
+
   // some tasks
 
   def log[F[_]](f: Logger[F] => F[Unit]): FtsWork[F] =
     FtsWork(ctx => f(ctx.logger))
 
-  def initialize[F[_]]: FtsWork[F] =
-    FtsWork(_.fts.initialize)
-
   def clearIndex[F[_]](coll: Option[Ident]): FtsWork[F] =
     coll match {
       case Some(cid) =>
@@ -40,7 +63,7 @@ object FtsWork {
         FtsWork(ctx => ctx.fts.clearAll(ctx.logger))
     }
 
-  def insertAll[F[_]: Effect](coll: Option[Ident]): FtsWork[F] =
+  def insertAll[F[_]: FlatMap](coll: Option[Ident]): FtsWork[F] =
     FtsWork
       .all(
         FtsWork(ctx =>

modules/joex/src/main/scala/docspell/joex/fts/Migration.scala

@@ -1,6 +1,6 @@
 package docspell.joex.fts
 
-import cats.Traverse
+import cats.{Applicative, FlatMap, Traverse}
 import cats.data.{Kleisli, OptionT}
 import cats.effect._
 import cats.implicits._
@@ -20,6 +20,9 @@ case class Migration[F[_]](
 
 object Migration {
 
+  def from[F[_]: Applicative: FlatMap](fm: FtsMigration[F]): Migration[F] =
+    Migration(fm.version, fm.engine, fm.description, FtsWork.from(fm.task))
+
   def apply[F[_]: Effect](
       cfg: Config.FullTextSearch,
       fts: FtsClient[F],

modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala

@@ -21,7 +21,7 @@ object MigrationTask {
       .flatMap(_ =>
         Task(ctx =>
           Migration[F](cfg, fts, ctx.store, ctx.logger)
-            .run(migrationTasks[F])
+            .run(migrationTasks[F](fts))
         )
       )
 
@@ -44,11 +44,7 @@ object MigrationTask {
       Some(DocspellSystem.migrationTaskTracker)
     )
 
-  private val solrEngine = Ident.unsafe("solr")
+  def migrationTasks[F[_]: Effect](fts: FtsClient[F]): List[Migration[F]] =
-  def migrationTasks[F[_]: Effect]: List[Migration[F]] =
+    fts.initialize.map(fm => Migration.from(fm))
-    List(
-      Migration[F](1, solrEngine, "initialize", FtsWork.initialize[F]),
-      Migration[F](2, solrEngine, "Index all from database", FtsWork.insertAll[F](None))
-    )
 
 }

modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala

@@ -21,13 +21,7 @@ object ReIndexTask {
     Task
       .log[F, Args](_.info(s"Running full-text re-index now"))
       .flatMap(_ =>
-        Task(ctx =>
+        Task(ctx => clearData[F](ctx.args.collective).forContext(cfg, fts).run(ctx))
-          (clearData[F](ctx.args.collective) ++
-            FtsWork.log[F](_.info("Inserting data from database")) ++
-            FtsWork.insertAll[F](
-              ctx.args.collective
-            )).forContext(cfg, fts).run(ctx)
-        )
       )
 
   def onCancel[F[_]: Sync]: Task[F, Args, Unit] =
@@ -41,7 +35,9 @@ object ReIndexTask {
             .clearIndex(collective)
             .recoverWith(
               FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
-            )
+            ) ++
+            FtsWork.log[F](_.info("Inserting data from database")) ++
+            FtsWork.insertAll[F](collective)
 
         case None =>
           FtsWork
@@ -50,6 +46,6 @@ object ReIndexTask {
               FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
             ) ++
             FtsWork.log[F](_.info("Running index initialize")) ++
-            FtsWork.initialize[F]
+            FtsWork.allInitializeTasks[F]
       })
 }