Refactor full-text migrations and add folder to solr schema

modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala

@@ -1,9 +1,8 @@
 package docspell.joex.fts
 
 import cats.data.{Kleisli, NonEmptyList}
-import cats.effect._
 import cats.implicits._
-import cats.{ApplicativeError, FlatMap, Semigroup}
+import cats.{Applicative, ApplicativeError, FlatMap, Monad, Semigroup}
 
 import docspell.common._
 import docspell.ftsclient._
@@ -15,6 +14,19 @@ object FtsWork {
   def apply[F[_]](f: FtsContext[F] => F[Unit]): FtsWork[F] =
     Kleisli(f)
 
+  def allInitializeTasks[F[_]: Monad]: FtsWork[F] =
+    FtsWork[F](_ => ().pure[F]).tap[FtsContext[F]].flatMap { ctx =>
+      NonEmptyList.fromList(ctx.fts.initialize.map(fm => from[F](fm.task))) match {
+        case Some(nel) =>
+          nel.reduce(semigroup[F])
+        case None =>
+          FtsWork[F](_ => ().pure[F])
+      }
+    }
+
+  def from[F[_]: FlatMap: Applicative](t: F[FtsMigration.Result]): FtsWork[F] =
+    Kleisli.liftF(t).flatMap(transformResult[F])
+
   def all[F[_]: FlatMap](
       m0: FtsWork[F],
       mn: FtsWork[F]*
@@ -24,14 +36,25 @@ object FtsWork {
   implicit def semigroup[F[_]: FlatMap]: Semigroup[FtsWork[F]] =
     Semigroup.instance((mt1, mt2) => mt1.flatMap(_ => mt2))
 
+  private def transformResult[F[_]: Applicative: FlatMap](
+      r: FtsMigration.Result
+  ): FtsWork[F] =
+    r match {
+      case FtsMigration.Result.WorkDone =>
+        Kleisli.pure(())
+
+      case FtsMigration.Result.IndexAll =>
+        insertAll[F](None)
+
+      case FtsMigration.Result.ReIndexAll =>
+        clearIndex[F](None) >> insertAll[F](None)
+    }
+
   // some tasks
 
   def log[F[_]](f: Logger[F] => F[Unit]): FtsWork[F] =
     FtsWork(ctx => f(ctx.logger))
 
-  def initialize[F[_]]: FtsWork[F] =
-    FtsWork(_.fts.initialize)
-
   def clearIndex[F[_]](coll: Option[Ident]): FtsWork[F] =
     coll match {
       case Some(cid) =>
@@ -40,7 +63,7 @@ object FtsWork {
         FtsWork(ctx => ctx.fts.clearAll(ctx.logger))
     }
 
-  def insertAll[F[_]: Effect](coll: Option[Ident]): FtsWork[F] =
+  def insertAll[F[_]: FlatMap](coll: Option[Ident]): FtsWork[F] =
     FtsWork
       .all(
         FtsWork(ctx =>

modules/joex/src/main/scala/docspell/joex/fts/Migration.scala

@@ -1,6 +1,6 @@
 package docspell.joex.fts
 
-import cats.Traverse
+import cats.{Applicative, FlatMap, Traverse}
 import cats.data.{Kleisli, OptionT}
 import cats.effect._
 import cats.implicits._
@@ -20,6 +20,9 @@ case class Migration[F[_]](
 
 object Migration {
 
+  def from[F[_]: Applicative: FlatMap](fm: FtsMigration[F]): Migration[F] =
+    Migration(fm.version, fm.engine, fm.description, FtsWork.from(fm.task))
+
   def apply[F[_]: Effect](
       cfg: Config.FullTextSearch,
       fts: FtsClient[F],

modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala

@@ -21,7 +21,7 @@ object MigrationTask {
       .flatMap(_ =>
         Task(ctx =>
           Migration[F](cfg, fts, ctx.store, ctx.logger)
-            .run(migrationTasks[F])
+            .run(migrationTasks[F](fts))
         )
       )
 
@@ -44,11 +44,7 @@ object MigrationTask {
       Some(DocspellSystem.migrationTaskTracker)
     )
 
-  private val solrEngine = Ident.unsafe("solr")
-  def migrationTasks[F[_]: Effect]: List[Migration[F]] =
-    List(
-      Migration[F](1, solrEngine, "initialize", FtsWork.initialize[F]),
-      Migration[F](2, solrEngine, "Index all from database", FtsWork.insertAll[F](None))
-    )
+  def migrationTasks[F[_]: Effect](fts: FtsClient[F]): List[Migration[F]] =
+    fts.initialize.map(fm => Migration.from(fm))
 
 }

modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala

@@ -21,13 +21,7 @@ object ReIndexTask {
     Task
       .log[F, Args](_.info(s"Running full-text re-index now"))
       .flatMap(_ =>
-        Task(ctx =>
-          (clearData[F](ctx.args.collective) ++
-            FtsWork.log[F](_.info("Inserting data from database")) ++
-            FtsWork.insertAll[F](
-              ctx.args.collective
-            )).forContext(cfg, fts).run(ctx)
-        )
+        Task(ctx => clearData[F](ctx.args.collective).forContext(cfg, fts).run(ctx))
       )
 
   def onCancel[F[_]: Sync]: Task[F, Args, Unit] =
@@ -41,7 +35,9 @@ object ReIndexTask {
             .clearIndex(collective)
             .recoverWith(
               FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
-            )
+            ) ++
+            FtsWork.log[F](_.info("Inserting data from database")) ++
+            FtsWork.insertAll[F](collective)
 
         case None =>
           FtsWork
@@ -50,6 +46,6 @@ object ReIndexTask {
               FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
             ) ++
             FtsWork.log[F](_.info("Running index initialize")) ++
-            FtsWork.initialize[F]
+            FtsWork.allInitializeTasks[F]
       })
 }