diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index b1baedcb..03f0e23b 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -11,14 +11,15 @@ import cats.implicits._ import docspell.backend.signup.{Config => SignupConfig} import docspell.common._ -import docspell.store.JdbcConfig import docspell.store.file.FileRepositoryConfig +import docspell.store.{JdbcConfig, SchemaMigrateConfig} import emil.javamail.Settings case class Config( mailDebug: Boolean, jdbc: JdbcConfig, + databaseSchema: SchemaMigrateConfig, signup: SignupConfig, files: Config.Files, addons: Config.Addons diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index a0c9e9b7..1cda608a 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -55,6 +55,20 @@ docspell.joex { password = "" } + # Additional settings related to schema migration. + database-schema = { + # Whether to run main database migrations. + run-main-migrations = true + + # Whether to run the fixup migrations. + run-fixup-migrations = true + + # Use with care. This repairs all migrations in the database by + # updating their checksums and removing failed migrations. Good + # for testing, not recommended for normal operation. + repair-schema = false + } + # Enable or disable debugging for e-mail related functionality. This # applies to both sending and receiving mails. For security reasons # logging is not very extensive on authentication failures. Setting diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index 646033c2..4601f9a0 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -25,7 +25,7 @@ import docspell.joex.updatecheck.UpdateCheckConfig import docspell.logging.LogConfig import docspell.pubsub.naive.PubSubConfig import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig} -import docspell.store.JdbcConfig +import docspell.store.{JdbcConfig, SchemaMigrateConfig} case class Config( appId: Ident, @@ -33,6 +33,7 @@ case class Config( logging: LogConfig, bind: Config.Bind, jdbc: JdbcConfig, + databaseSchema: SchemaMigrateConfig, scheduler: SchedulerConfig, periodicScheduler: PeriodicSchedulerConfig, userTasks: Config.UserTasks, diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index ae1bf1a7..a680844d 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -41,6 +41,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, + cfg.databaseSchema, cfg.files.defaultFileRepositoryConfig, pools.connectEC ) diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 2c6d4523..ccf076ad 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -387,6 +387,20 @@ docspell.server { password = "" } + # Additional settings related to schema migration. + database-schema = { + # Whether to run main database migrations. + run-main-migrations = true + + # Whether to run the fixup migrations. + run-fixup-migrations = true + + # Use with care. This repairs all migrations in the database by + # updating their checksums and removing failed migrations. Good + # for testing, not recommended for normal operation. + repair-schema = false + } + # Configuration for registering new users. signup { diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index e4c1a5bc..4eb88c30 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -83,6 +83,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, + cfg.backend.databaseSchema, cfg.backend.files.defaultFileRepositoryConfig, pools.connectEC ) diff --git a/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala b/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala new file mode 100644 index 00000000..8659ef99 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store + +case class SchemaMigrateConfig( + runMainMigrations: Boolean, + runFixupMigrations: Boolean, + repairSchema: Boolean +) + +object SchemaMigrateConfig { + val defaults = SchemaMigrateConfig(true, true, false) +} diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index d41e83d2..f1c327c0 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -42,6 +42,7 @@ object Store { def create[F[_]: Async]( jdbc: JdbcConfig, + schemaCfg: SchemaMigrateConfig, fileRepoConfig: FileRepositoryConfig, connectEC: ExecutionContext ): Resource[F, Store[F]] = { @@ -58,7 +59,7 @@ object Store { } xa = HikariTransactor(ds, connectEC) fr = FileRepository.apply(xa, ds, fileRepoConfig, true) - st = new StoreImpl[F](fr, jdbc, ds, xa) + st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa) _ <- Resource.eval(st.migrate) } yield st } diff --git a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala index d68ef6e3..f49b1035 100644 --- a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala +++ b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala @@ -13,9 +13,9 @@ import cats.effect.Async import cats.implicits._ import cats.~> +import docspell.store._ import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.migrate.FlywayMigrate -import docspell.store.{AddResult, JdbcConfig, Store} import doobie._ import doobie.implicits._ @@ -23,6 +23,7 @@ import doobie.implicits._ final class StoreImpl[F[_]: Async]( val fileRepo: FileRepository[F], jdbc: JdbcConfig, + schemaCfg: SchemaMigrateConfig, ds: DataSource, val transactor: Transactor[F] ) extends Store[F] { @@ -38,7 +39,7 @@ final class StoreImpl[F[_]: Async]( FunctionK.lift(transact) def migrate: F[Int] = - FlywayMigrate[F](jdbc, xa).run.map(_.migrationsExecuted) + FlywayMigrate[F](jdbc, schemaCfg, xa).run.map(_.migrationsExecuted) def transact[A](prg: ConnectionIO[A]): F[A] = prg.transact(xa) diff --git a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala index 5df5ba1d..e0b1e2d5 100644 --- a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala +++ b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala @@ -10,15 +10,19 @@ import cats.data.OptionT import cats.effect.Sync import cats.implicits._ -import docspell.store.JdbcConfig import docspell.store.migrate.FlywayMigrate.MigrationKind +import docspell.store.{JdbcConfig, SchemaMigrateConfig} import doobie.implicits._ import doobie.util.transactor.Transactor import org.flywaydb.core.Flyway import org.flywaydb.core.api.output.MigrateResult -class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) { +class FlywayMigrate[F[_]: Sync]( + jdbc: JdbcConfig, + cfg: SchemaMigrateConfig, + xa: Transactor[F] +) { private[this] val logger = docspell.logging.getLogger[F] private def createLocations(folder: String) = @@ -49,28 +53,46 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) { def run: F[MigrateResult] = for { _ <- runFixups - fw <- createFlyway(MigrationKind.Main) - _ <- logger.info(s"!!! Running main migrations") - result <- Sync[F].blocking(fw.migrate()) + result <- runMain } yield result + def runMain: F[MigrateResult] = + if (!cfg.runMainMigrations) + logger + .info("Running main migrations is disabled!") + .as(new MigrateResult("", "", "")) + else + for { + fw <- createFlyway(MigrationKind.Main) + _ <- logger.info(s"!!! Running main migrations (repair=${cfg.repairSchema})") + _ <- if (cfg.repairSchema) Sync[F].blocking(fw.repair()).void else ().pure[F] + result <- Sync[F].blocking(fw.migrate()) + } yield result + // A hack to fix already published migrations def runFixups: F[Unit] = - isSchemaEmpty.flatMap { - case true => - ().pure[F] - case false => - (for { - current <- OptionT(getSchemaVersion) - _ <- OptionT - .fromOption[F](versionComponents(current)) - .filter(v => v._1 >= 1 && v._2 >= 32) - fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups)) - _ <- OptionT.liftF(logger.info(s"!!! Running fixup migrations")) - _ <- OptionT.liftF(Sync[F].blocking(fw.migrate())) - } yield ()) - .getOrElseF(logger.info(s"Fixup migrations not applied.")) - } + if (!cfg.runFixupMigrations) logger.info(s"Running fixup migrations is disabled!") + else + isSchemaEmpty.flatMap { + case true => + ().pure[F] + case false => + (for { + current <- OptionT(getSchemaVersion) + _ <- OptionT + .fromOption[F](versionComponents(current)) + .filter(v => v._1 >= 1 && v._2 >= 32) + fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups)) + _ <- OptionT.liftF( + logger.info(s"!!! Running fixup migrations (repair=${cfg.repairSchema})") + ) + _ <- + if (cfg.repairSchema) OptionT.liftF(Sync[F].blocking(fw.repair()).void) + else OptionT.pure[F](()) + _ <- OptionT.liftF(Sync[F].blocking(fw.migrate())) + } yield ()) + .getOrElseF(logger.info(s"Fixup migrations not applied.")) + } private def isSchemaEmpty: F[Boolean] = sql"select count(1) from flyway_schema_history" @@ -95,8 +117,12 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) { } object FlywayMigrate { - def apply[F[_]: Sync](jdbcConfig: JdbcConfig, xa: Transactor[F]): FlywayMigrate[F] = - new FlywayMigrate[F](jdbcConfig, xa) + def apply[F[_]: Sync]( + jdbcConfig: JdbcConfig, + schemaCfg: SchemaMigrateConfig, + xa: Transactor[F] + ): FlywayMigrate[F] = + new FlywayMigrate[F](jdbcConfig, schemaCfg, xa) sealed trait MigrationKind { def table: String diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala index eca434ce..cbff6a1f 100644 --- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -22,13 +22,15 @@ import org.mariadb.jdbc.MariaDbDataSource import org.postgresql.ds.PGConnectionPoolDataSource trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite => + def schemaMigrateConfig = + StoreFixture.schemaMigrateConfig val xa = ResourceFixture { val cfg = StoreFixture.memoryDB("test") for { ds <- StoreFixture.dataSource(cfg) xa <- StoreFixture.makeXA(ds) - _ <- Resource.eval(FlywayMigrate[IO](cfg, xa).run) + _ <- Resource.eval(FlywayMigrate[IO](cfg, schemaMigrateConfig, xa).run) } yield xa } @@ -42,6 +44,7 @@ trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite => } object StoreFixture { + val schemaMigrateConfig = SchemaMigrateConfig.defaults def memoryDB(dbname: String): JdbcConfig = JdbcConfig( @@ -94,7 +97,7 @@ object StoreFixture { xa <- makeXA(ds) cfg = FileRepositoryConfig.Database(64 * 1024) fr = FileRepository[IO](xa, ds, cfg, true) - store = new StoreImpl[IO](fr, jdbc, ds, xa) + store = new StoreImpl[IO](fr, jdbc, schemaMigrateConfig, ds, xa) _ <- Resource.eval(store.migrate) } yield store diff --git a/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala index 4e47d5c2..cb51023a 100644 --- a/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala +++ b/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala @@ -10,7 +10,7 @@ import cats.effect.IO import cats.effect.unsafe.implicits._ import docspell.logging.TestLoggingConfig -import docspell.store.StoreFixture +import docspell.store.{SchemaMigrateConfig, StoreFixture} import munit.FunSuite @@ -21,7 +21,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig { val ds = StoreFixture.dataSource(jdbc) val result = ds.flatMap(StoreFixture.makeXA).use { xa => - FlywayMigrate[IO](jdbc, xa).run + FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run } assert(result.unsafeRunSync().migrationsExecuted > 0) @@ -40,7 +40,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig { val result = ds.flatMap(StoreFixture.makeXA).use { xa => - FlywayMigrate[IO](jdbc, xa).run + FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run } result.unsafeRunSync() diff --git a/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala index a77382dc..75e765b6 100644 --- a/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala +++ b/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala @@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._ import docspell.common.LenientUri import docspell.logging.TestLoggingConfig -import docspell.store.{JdbcConfig, StoreFixture} +import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture} import com.dimafeng.testcontainers.MariaDBContainer import com.dimafeng.testcontainers.munit.TestContainerForAll @@ -32,7 +32,7 @@ class MariaDbMigrateTest JdbcConfig(LenientUri.unsafe(cnt.jdbcUrl), cnt.dbUsername, cnt.dbPassword) val ds = StoreFixture.dataSource(jdbc) val result = ds.flatMap(StoreFixture.makeXA).use { xa => - FlywayMigrate[IO](jdbc, xa).run + FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run } assert(result.unsafeRunSync().migrationsExecuted > 0) // a second time to apply fixup migrations diff --git a/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala index 235b240e..f71d01c8 100644 --- a/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala +++ b/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala @@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._ import docspell.common.LenientUri import docspell.logging.TestLoggingConfig -import docspell.store.{JdbcConfig, StoreFixture} +import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture} import com.dimafeng.testcontainers.PostgreSQLContainer import com.dimafeng.testcontainers.munit.TestContainerForAll @@ -34,7 +34,7 @@ class PostgresqlMigrateTest val ds = StoreFixture.dataSource(jdbc) val result = ds.flatMap(StoreFixture.makeXA).use { xa => - FlywayMigrate[IO](jdbc, xa).run + FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run } assert(result.unsafeRunSync().migrationsExecuted > 0)