From aeba4ba9137af6e0357f33a35d0b5ba375626b2f Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 12 Jul 2020 00:30:37 +0200 Subject: [PATCH] Refactor full-text migrations and add folder to solr schema --- .../scala/docspell/ftsclient/FtsClient.scala | 9 ++--- .../docspell/ftsclient/FtsMigration.scala | 24 +++++++++++++ .../main/scala/docspell/ftssolr/Field.scala | 1 + .../docspell/ftssolr/SolrFtsClient.scala | 2 +- .../scala/docspell/ftssolr/SolrSetup.scala | 32 +++++++++++++++-- .../scala/docspell/joex/fts/FtsWork.scala | 35 +++++++++++++++---- .../scala/docspell/joex/fts/Migration.scala | 5 ++- .../docspell/joex/fts/MigrationTask.scala | 10 ++---- .../scala/docspell/joex/fts/ReIndexTask.scala | 14 +++----- 9 files changed, 102 insertions(+), 30 deletions(-) create mode 100644 modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala diff --git a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala index 3a0e3d17..b3bdcf9a 100644 --- a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala +++ b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala @@ -17,11 +17,12 @@ import org.log4s.getLogger */ trait FtsClient[F[_]] { - /** Initialization tasks. This is called exactly once and then never + /** Initialization tasks. This is called exactly once at the very + * beginning when initializing the full-text index and then never * again (except when re-indexing everything). It may be used to * setup the database. */ - def initialize: F[Unit] + def initialize: List[FtsMigration[F]] /** Run a full-text search. */ def search(q: FtsQuery): F[FtsResult] @@ -107,8 +108,8 @@ object FtsClient { new FtsClient[F] { private[this] val logger = Logger.log4s[F](getLogger) - def initialize: F[Unit] = - logger.info("Full-text search is disabled!") + def initialize: List[FtsMigration[F]] = + Nil def search(q: FtsQuery): F[FtsResult] = logger.warn("Full-text search is disabled!") *> FtsResult.empty.pure[F] diff --git a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala new file mode 100644 index 00000000..3e8fae4e --- /dev/null +++ b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala @@ -0,0 +1,24 @@ +package docspell.ftsclient + +import docspell.common._ + +final case class FtsMigration[F[_]]( + version: Int, + engine: Ident, + description: String, + task: F[FtsMigration.Result] +) + +object FtsMigration { + + sealed trait Result + object Result { + case object WorkDone extends Result + case object ReIndexAll extends Result + case object IndexAll extends Result + + def workDone: Result = WorkDone + def reIndexAll: Result = ReIndexAll + def indexAll: Result = IndexAll + } +} diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala index 053eb5c8..6031cd61 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala @@ -25,6 +25,7 @@ object Field { val content_en = Field("content_en") val itemName = Field("itemName") val itemNotes = Field("itemNotes") + val folderId = Field("folder") def contentField(lang: Language): Field = lang match { diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala index 635c0d97..c0994328 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala @@ -17,7 +17,7 @@ final class SolrFtsClient[F[_]: Effect]( solrQuery: SolrQuery[F] ) extends FtsClient[F] { - def initialize: F[Unit] = + def initialize: List[FtsMigration[F]] = solrSetup.setupSchema def search(q: FtsQuery): F[FtsResult] = diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala index 6952c823..932519c8 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala @@ -4,6 +4,7 @@ import cats.effect._ import cats.implicits._ import docspell.common._ +import docspell.ftsclient.FtsMigration import _root_.io.circe._ import _root_.io.circe.generic.semiauto._ @@ -15,21 +16,48 @@ import org.http4s.client.dsl.Http4sClientDsl trait SolrSetup[F[_]] { - def setupSchema: F[Unit] + def setupSchema: List[FtsMigration[F]] } object SolrSetup { + private val solrEngine = Ident.unsafe("solr") def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrSetup[F] = { val dsl = new Http4sClientDsl[F] {} import dsl._ new SolrSetup[F] { + val url = (Uri.unsafeFromString(cfg.url.asString) / "schema") .withQueryParam("commitWithin", cfg.commitWithin.toString) - def setupSchema: F[Unit] = { + def setupSchema: List[FtsMigration[F]] = + List( + FtsMigration[F]( + 1, + solrEngine, + "Initialize", + setupCoreSchema.map(_ => FtsMigration.Result.workDone) + ), + FtsMigration[F]( + 3, + solrEngine, + "Add folder field", + addFolderField.map(_ => FtsMigration.Result.workDone) + ), + FtsMigration[F]( + 4, + solrEngine, + "Index all from database", + FtsMigration.Result.indexAll.pure[F] + ) + ) + + def addFolderField: F[Unit] = + addStringField(Field.folderId) + + def setupCoreSchema: F[Unit] = { val cmds0 = List( Field.id, diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala index a4952271..fc3c77b3 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala @@ -1,9 +1,8 @@ package docspell.joex.fts import cats.data.{Kleisli, NonEmptyList} -import cats.effect._ import cats.implicits._ -import cats.{ApplicativeError, FlatMap, Semigroup} +import cats.{Applicative, ApplicativeError, FlatMap, Monad, Semigroup} import docspell.common._ import docspell.ftsclient._ @@ -15,6 +14,19 @@ object FtsWork { def apply[F[_]](f: FtsContext[F] => F[Unit]): FtsWork[F] = Kleisli(f) + def allInitializeTasks[F[_]: Monad]: FtsWork[F] = + FtsWork[F](_ => ().pure[F]).tap[FtsContext[F]].flatMap { ctx => + NonEmptyList.fromList(ctx.fts.initialize.map(fm => from[F](fm.task))) match { + case Some(nel) => + nel.reduce(semigroup[F]) + case None => + FtsWork[F](_ => ().pure[F]) + } + } + + def from[F[_]: FlatMap: Applicative](t: F[FtsMigration.Result]): FtsWork[F] = + Kleisli.liftF(t).flatMap(transformResult[F]) + def all[F[_]: FlatMap]( m0: FtsWork[F], mn: FtsWork[F]* @@ -24,14 +36,25 @@ object FtsWork { implicit def semigroup[F[_]: FlatMap]: Semigroup[FtsWork[F]] = Semigroup.instance((mt1, mt2) => mt1.flatMap(_ => mt2)) + private def transformResult[F[_]: Applicative: FlatMap]( + r: FtsMigration.Result + ): FtsWork[F] = + r match { + case FtsMigration.Result.WorkDone => + Kleisli.pure(()) + + case FtsMigration.Result.IndexAll => + insertAll[F](None) + + case FtsMigration.Result.ReIndexAll => + clearIndex[F](None) >> insertAll[F](None) + } + // some tasks def log[F[_]](f: Logger[F] => F[Unit]): FtsWork[F] = FtsWork(ctx => f(ctx.logger)) - def initialize[F[_]]: FtsWork[F] = - FtsWork(_.fts.initialize) - def clearIndex[F[_]](coll: Option[Ident]): FtsWork[F] = coll match { case Some(cid) => @@ -40,7 +63,7 @@ object FtsWork { FtsWork(ctx => ctx.fts.clearAll(ctx.logger)) } - def insertAll[F[_]: Effect](coll: Option[Ident]): FtsWork[F] = + def insertAll[F[_]: FlatMap](coll: Option[Ident]): FtsWork[F] = FtsWork .all( FtsWork(ctx => diff --git a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala index 4eb7df6c..40c5bf4a 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala @@ -1,6 +1,6 @@ package docspell.joex.fts -import cats.Traverse +import cats.{Applicative, FlatMap, Traverse} import cats.data.{Kleisli, OptionT} import cats.effect._ import cats.implicits._ @@ -20,6 +20,9 @@ case class Migration[F[_]]( object Migration { + def from[F[_]: Applicative: FlatMap](fm: FtsMigration[F]): Migration[F] = + Migration(fm.version, fm.engine, fm.description, FtsWork.from(fm.task)) + def apply[F[_]: Effect]( cfg: Config.FullTextSearch, fts: FtsClient[F], diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala index 4189fc25..b8b27b5e 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -21,7 +21,7 @@ object MigrationTask { .flatMap(_ => Task(ctx => Migration[F](cfg, fts, ctx.store, ctx.logger) - .run(migrationTasks[F]) + .run(migrationTasks[F](fts)) ) ) @@ -44,11 +44,7 @@ object MigrationTask { Some(DocspellSystem.migrationTaskTracker) ) - private val solrEngine = Ident.unsafe("solr") - def migrationTasks[F[_]: Effect]: List[Migration[F]] = - List( - Migration[F](1, solrEngine, "initialize", FtsWork.initialize[F]), - Migration[F](2, solrEngine, "Index all from database", FtsWork.insertAll[F](None)) - ) + def migrationTasks[F[_]: Effect](fts: FtsClient[F]): List[Migration[F]] = + fts.initialize.map(fm => Migration.from(fm)) } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index 205f31c8..c1d794e4 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -21,13 +21,7 @@ object ReIndexTask { Task .log[F, Args](_.info(s"Running full-text re-index now")) .flatMap(_ => - Task(ctx => - (clearData[F](ctx.args.collective) ++ - FtsWork.log[F](_.info("Inserting data from database")) ++ - FtsWork.insertAll[F]( - ctx.args.collective - )).forContext(cfg, fts).run(ctx) - ) + Task(ctx => clearData[F](ctx.args.collective).forContext(cfg, fts).run(ctx)) ) def onCancel[F[_]: Sync]: Task[F, Args, Unit] = @@ -41,7 +35,9 @@ object ReIndexTask { .clearIndex(collective) .recoverWith( FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing.")) - ) + ) ++ + FtsWork.log[F](_.info("Inserting data from database")) ++ + FtsWork.insertAll[F](collective) case None => FtsWork @@ -50,6 +46,6 @@ object ReIndexTask { FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing.")) ) ++ FtsWork.log[F](_.info("Running index initialize")) ++ - FtsWork.initialize[F] + FtsWork.allInitializeTasks[F] }) }