From 360cad3304af85fa56fccc20605a4d481c1cf6fe Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 17 Jan 2021 11:19:27 +0100 Subject: [PATCH] Refactoring solr/fts migration When re-indexing everything, skip intermediate populating the index and do this as the very last step. Parameterize adding new fields by their language. --- .../docspell/ftsclient/FtsMigration.scala | 11 ++++++- .../scala/docspell/ftssolr/SolrSetup.scala | 30 ++++--------------- .../scala/docspell/joex/fts/FtsWork.scala | 20 +++++++++---- .../scala/docspell/joex/fts/Migration.scala | 5 ++++ .../scala/docspell/joex/fts/ReIndexTask.scala | 2 +- .../scala/docspell/joex/fts/package.scala | 3 ++ 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala index 3e8fae4e..22858c19 100644 --- a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala +++ b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala @@ -1,5 +1,8 @@ package docspell.ftsclient +import cats.Functor +import cats.implicits._ + import docspell.common._ final case class FtsMigration[F[_]]( @@ -7,7 +10,13 @@ final case class FtsMigration[F[_]]( engine: Ident, description: String, task: F[FtsMigration.Result] -) +) { + + def changeResult(f: FtsMigration.Result => FtsMigration.Result)(implicit + F: Functor[F] + ): FtsMigration[F] = + copy(task = task.map(f)) +} object FtsMigration { diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala index 769919bd..fb31d912 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala @@ -126,19 +126,10 @@ object SolrSetup { lang match { case None => run(DeleteField.command(DeleteField(field))).attempt *> - run(AddField.command(AddField.text(field))) - case Some(Language.German) => + run(AddField.command(AddField.textGeneral(field))) + case Some(lang) => run(DeleteField.command(DeleteField(field))).attempt *> - run(AddField.command(AddField.textDE(field))) - case Some(Language.English) => - run(DeleteField.command(DeleteField(field))).attempt *> - run(AddField.command(AddField.textEN(field))) - case Some(Language.French) => - run(DeleteField.command(DeleteField(field))).attempt *> - run(AddField.command(AddField.textFR(field))) - case Some(Language.Italian) => - run(DeleteField.command(DeleteField(field))).attempt *> - run(AddField.command(AddField.textIT(field))) + run(AddField.command(AddField.textLang(field, lang))) } } } @@ -164,20 +155,11 @@ object SolrSetup { def string(field: Field): AddField = AddField(field, "string", true, true, false) - def text(field: Field): AddField = + def textGeneral(field: Field): AddField = AddField(field, "text_general", true, true, false) - def textDE(field: Field): AddField = - AddField(field, "text_de", true, true, false) - - def textEN(field: Field): AddField = - AddField(field, "text_en", true, true, false) - - def textFR(field: Field): AddField = - AddField(field, "text_fr", true, true, false) - - def textIT(field: Field): AddField = - AddField(field, "text_it", true, true, false) + def textLang(field: Field, lang: Language): AddField = + AddField(field, s"text_${lang.iso2}", true, true, false) } case class DeleteField(name: Field) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala index 88369f9f..7ddfa99d 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala @@ -14,16 +14,26 @@ object FtsWork { def apply[F[_]](f: FtsContext[F] => F[Unit]): FtsWork[F] = Kleisli(f) - def allInitializeTasks[F[_]: Monad]: FtsWork[F] = - FtsWork[F](_ => ().pure[F]).tap[FtsContext[F]].flatMap { ctx => - NonEmptyList.fromList(ctx.fts.initialize.map(fm => from[F](fm.task))) match { + /** Runs all migration tasks unconditionally and inserts all data as last step. */ + def reInitializeTasks[F[_]: Monad]: FtsWork[F] = + FtsWork { ctx => + val migrations = + ctx.fts.initialize.map(fm => fm.changeResult(_ => FtsMigration.Result.workDone)) + + NonEmptyList.fromList(migrations) match { case Some(nel) => - nel.reduce(semigroup[F]) + nel + .map(fm => from[F](fm.task)) + .append(insertAll[F](None)) + .reduce(semigroup[F]) + .run(ctx) case None => - FtsWork[F](_ => ().pure[F]) + ().pure[F] } } + /** + */ def from[F[_]: FlatMap: Applicative](t: F[FtsMigration.Result]): FtsWork[F] = Kleisli.liftF(t).flatMap(transformResult[F]) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala index c47d4308..5ad9d028 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala @@ -11,6 +11,11 @@ import docspell.joex.Config import docspell.store.records.RFtsMigration import docspell.store.{AddResult, Store} +/** Migrating the index from the previous version to this version. + * + * The sql database stores the outcome of a migration task. If this + * task has already been applied, it is skipped. + */ case class Migration[F[_]]( version: Int, engine: Ident, diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index c1d794e4..5dd45943 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -46,6 +46,6 @@ object ReIndexTask { FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing.")) ) ++ FtsWork.log[F](_.info("Running index initialize")) ++ - FtsWork.allInitializeTasks[F] + FtsWork.reInitializeTasks[F] }) } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/package.scala b/modules/joex/src/main/scala/docspell/joex/fts/package.scala index 784754ab..7cf8de80 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/package.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/package.scala @@ -4,6 +4,9 @@ import cats.data.Kleisli package object fts { + /** Some work that must be done to advance the schema of the fulltext + * index. + */ type FtsWork[F[_]] = Kleisli[F, FtsContext[F], Unit] }