From 2f6e531c45f4c7bdfcade164329dda94ad69a218 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 21 Jun 2020 01:37:23 +0200 Subject: [PATCH] Refactoring index migration task --- .../scala/docspell/joex/fts/IndexTask.scala | 9 ++- .../scala/docspell/joex/fts/MigrateCtx.scala | 13 ++++ .../scala/docspell/joex/fts/Migration.scala | 66 ++----------------- .../docspell/joex/fts/MigrationTask.scala | 56 ++++++++++++++++ .../scala/docspell/joex/fts/package.scala | 9 +++ 5 files changed, 93 insertions(+), 60 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/fts/MigrateCtx.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/fts/package.scala diff --git a/modules/joex/src/main/scala/docspell/joex/fts/IndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/IndexTask.scala index c7298943..70e01631 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/IndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/IndexTask.scala @@ -22,7 +22,7 @@ object IndexTask { .flatMap(_ => Task(ctx => Migration[F](cfg, ctx.store, fts, ctx.logger) - .run(Migration.migrationTasks[F]) + .run(migrationTasks[F]) ) ) @@ -45,4 +45,11 @@ object IndexTask { Some(taskName) ) + private val solrEngine = Ident.unsafe("solr") + def migrationTasks[F[_]: Effect]: List[Migration[F]] = + List( + Migration[F](1, solrEngine, "initialize", MigrationTask[F](ctx => ctx.fts.initialize)), + Migration[F](2, solrEngine, "Index all from database", MigrationTask.insertAll[F]) + ) + } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrateCtx.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrateCtx.scala new file mode 100644 index 00000000..99d5d367 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrateCtx.scala @@ -0,0 +1,13 @@ +package docspell.joex.fts + +import docspell.common.Logger +import docspell.joex.Config +import docspell.store.Store +import docspell.ftsclient.FtsClient + +case class MigrateCtx[F[_]]( + cfg: Config.FullTextSearch, + store: Store[F], + fts: FtsClient[F], + logger: Logger[F] +) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala index c546e690..0c50eba8 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/Migration.scala @@ -8,25 +8,16 @@ import docspell.common._ import docspell.joex.Config import docspell.store.{AddResult, Store} import docspell.store.records.RFtsMigration -import docspell.store.queries.{QAttachment, QItem} import docspell.ftsclient._ +case class Migration[F[_]]( + version: Int, + engine: Ident, + description: String, + task: MigrationTask[F] +) + object Migration { - private val solrEngine = Ident.unsafe("solr") - - case class MigrateCtx[F[_]]( - cfg: Config.FullTextSearch, - store: Store[F], - fts: FtsClient[F], - logger: Logger[F] - ) - - case class Migration[F[_]]( - version: Int, - engine: Ident, - description: String, - task: Kleisli[F, MigrateCtx[F], Unit] - ) def apply[F[_]: Effect]( cfg: Config.FullTextSearch, @@ -72,47 +63,4 @@ object Migration { ctx.logger.info(s"Migration ${m.version}/${m.description} already applied.") ) } - - def migrationTasks[F[_]: Effect]: List[Migration[F]] = - List( - Migration[F](1, solrEngine, "initialize", Kleisli(ctx => ctx.fts.initialize)), - Migration[F]( - 2, - solrEngine, - "Index all attachments from database", - Kleisli(ctx => - ctx.fts.indexData( - ctx.logger, - ctx.store - .transact( - QAttachment.allAttachmentMetaAndName(ctx.cfg.migration.indexAllChunk) - ) - .map(caa => - TextData - .attachment( - caa.item, - caa.id, - caa.collective, - caa.lang, - caa.name, - caa.content - ) - ) - ) - ) - ), - Migration[F]( - 3, - solrEngine, - "Index all items from database", - Kleisli(ctx => - ctx.fts.indexData( - ctx.logger, - ctx.store - .transact(QItem.allNameAndNotes(ctx.cfg.migration.indexAllChunk * 5)) - .map(nn => TextData.item(nn.id, nn.collective, Option(nn.name), nn.notes)) - ) - ) - ) - ) } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala new file mode 100644 index 00000000..a2a6a671 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -0,0 +1,56 @@ +package docspell.joex.fts + +import cats.effect._ +import cats.data.{Kleisli, NonEmptyList} +import cats.{FlatMap, Semigroup} +import docspell.store.queries.{QAttachment, QItem} +import docspell.ftsclient.TextData + +object MigrationTask { + def apply[F[_]](f: MigrateCtx[F] => F[Unit]): MigrationTask[F] = + Kleisli(f) + + def all[F[_]: FlatMap]( + m0: MigrationTask[F], + mn: MigrationTask[F]* + ): MigrationTask[F] = + NonEmptyList.of(m0, mn: _*).reduce(semigroup[F]) + + implicit def semigroup[F[_]: FlatMap]: Semigroup[MigrationTask[F]] = + Semigroup.instance((mt1, mt2) => mt1.flatMap(_ => mt2)) + + // some tasks + + def insertAll[F[_]: Effect]: MigrationTask[F] = + MigrationTask + .all( + MigrationTask(ctx => + ctx.fts.indexData( + ctx.logger, + ctx.store + .transact( + QAttachment.allAttachmentMetaAndName(ctx.cfg.migration.indexAllChunk) + ) + .map(caa => + TextData + .attachment( + caa.item, + caa.id, + caa.collective, + caa.lang, + caa.name, + caa.content + ) + ) + ) + ), + MigrationTask(ctx => + ctx.fts.indexData( + ctx.logger, + ctx.store + .transact(QItem.allNameAndNotes(ctx.cfg.migration.indexAllChunk * 5)) + .map(nn => TextData.item(nn.id, nn.collective, Option(nn.name), nn.notes)) + ) + ) + ) +} diff --git a/modules/joex/src/main/scala/docspell/joex/fts/package.scala b/modules/joex/src/main/scala/docspell/joex/fts/package.scala new file mode 100644 index 00000000..80b2ef26 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/fts/package.scala @@ -0,0 +1,9 @@ +package docspell.joex + +import cats.data.Kleisli + +package object fts { + + type MigrationTask[F[_]] = Kleisli[F, MigrateCtx[F], Unit] + +}