Merge pull request #1563 from eikek/flyway-repair

Configure run/repair db migrations

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -11,14 +11,15 @@ import cats.implicits._
 
 import docspell.backend.signup.{Config => SignupConfig}
 import docspell.common._
-import docspell.store.JdbcConfig
 import docspell.store.file.FileRepositoryConfig
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
 
 import emil.javamail.Settings
 
 case class Config(
     mailDebug: Boolean,
     jdbc: JdbcConfig,
+    databaseSchema: SchemaMigrateConfig,
     signup: SignupConfig,
     files: Config.Files,
     addons: Config.Addons

modules/joex/src/main/resources/reference.conf

@@ -55,6 +55,20 @@ docspell.joex {
     password = ""
   }
 
+  # Additional settings related to schema migration.
+  database-schema = {
+    # Whether to run main database migrations.
+    run-main-migrations = true
+
+    # Whether to run the fixup migrations.
+    run-fixup-migrations = true
+
+    # Use with care. This repairs all migrations in the database by
+    # updating their checksums and removing failed migrations. Good
+    # for testing, not recommended for normal operation.
+    repair-schema = false
+  }
+
   # Enable or disable debugging for e-mail related functionality. This
   # applies to both sending and receiving mails. For security reasons
   # logging is not very extensive on authentication failures. Setting

modules/joex/src/main/scala/docspell/joex/Config.scala

@@ -25,7 +25,7 @@ import docspell.joex.updatecheck.UpdateCheckConfig
 import docspell.logging.LogConfig
 import docspell.pubsub.naive.PubSubConfig
 import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
-import docspell.store.JdbcConfig
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
 
 case class Config(
     appId: Ident,
@@ -33,6 +33,7 @@ case class Config(
     logging: LogConfig,
     bind: Config.Bind,
     jdbc: JdbcConfig,
+    databaseSchema: SchemaMigrateConfig,
     scheduler: SchedulerConfig,
     periodicScheduler: PeriodicSchedulerConfig,
     userTasks: Config.UserTasks,

modules/joex/src/main/scala/docspell/joex/JoexServer.scala

@@ -41,6 +41,7 @@ object JoexServer {
 
       store <- Store.create[F](
         cfg.jdbc,
+        cfg.databaseSchema,
         cfg.files.defaultFileRepositoryConfig,
         pools.connectEC
       )

modules/restserver/src/main/resources/reference.conf

@@ -387,6 +387,20 @@ docspell.server {
       password = ""
     }
 
+    # Additional settings related to schema migration.
+    database-schema = {
+      # Whether to run main database migrations.
+      run-main-migrations = true
+
+      # Whether to run the fixup migrations.
+      run-fixup-migrations = true
+
+      # Use with care. This repairs all migrations in the database by
+      # updating their checksums and removing failed migrations. Good
+      # for testing, not recommended for normal operation.
+      repair-schema = false
+    }
+
     # Configuration for registering new users.
     signup {
 

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -83,6 +83,7 @@ object RestServer {
       httpClient <- BlazeClientBuilder[F].resource
       store <- Store.create[F](
         cfg.backend.jdbc,
+        cfg.backend.databaseSchema,
         cfg.backend.files.defaultFileRepositoryConfig,
         pools.connectEC
       )

modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store
+
+case class SchemaMigrateConfig(
+    runMainMigrations: Boolean,
+    runFixupMigrations: Boolean,
+    repairSchema: Boolean
+)
+
+object SchemaMigrateConfig {
+  val defaults = SchemaMigrateConfig(true, true, false)
+}

modules/store/src/main/scala/docspell/store/Store.scala

@@ -42,6 +42,7 @@ object Store {
 
   def create[F[_]: Async](
       jdbc: JdbcConfig,
+      schemaCfg: SchemaMigrateConfig,
       fileRepoConfig: FileRepositoryConfig,
       connectEC: ExecutionContext
   ): Resource[F, Store[F]] = {
@@ -58,7 +59,7 @@ object Store {
       }
       xa = HikariTransactor(ds, connectEC)
       fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
-      st = new StoreImpl[F](fr, jdbc, ds, xa)
+      st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa)
       _ <- Resource.eval(st.migrate)
     } yield st
   }

modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala

@@ -13,9 +13,9 @@ import cats.effect.Async
 import cats.implicits._
 import cats.~>
 
+import docspell.store._
 import docspell.store.file.{FileRepository, FileRepositoryConfig}
 import docspell.store.migrate.FlywayMigrate
-import docspell.store.{AddResult, JdbcConfig, Store}
 
 import doobie._
 import doobie.implicits._
@@ -23,6 +23,7 @@ import doobie.implicits._
 final class StoreImpl[F[_]: Async](
     val fileRepo: FileRepository[F],
     jdbc: JdbcConfig,
+    schemaCfg: SchemaMigrateConfig,
     ds: DataSource,
     val transactor: Transactor[F]
 ) extends Store[F] {
@@ -38,7 +39,7 @@ final class StoreImpl[F[_]: Async](
     FunctionK.lift(transact)
 
   def migrate: F[Int] =
-    FlywayMigrate[F](jdbc, xa).run.map(_.migrationsExecuted)
+    FlywayMigrate[F](jdbc, schemaCfg, xa).run.map(_.migrationsExecuted)
 
   def transact[A](prg: ConnectionIO[A]): F[A] =
     prg.transact(xa)

modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala

@@ -10,15 +10,19 @@ import cats.data.OptionT
 import cats.effect.Sync
 import cats.implicits._
 
-import docspell.store.JdbcConfig
 import docspell.store.migrate.FlywayMigrate.MigrationKind
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
 
 import doobie.implicits._
 import doobie.util.transactor.Transactor
 import org.flywaydb.core.Flyway
 import org.flywaydb.core.api.output.MigrateResult
 
-class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
+class FlywayMigrate[F[_]: Sync](
+    jdbc: JdbcConfig,
+    cfg: SchemaMigrateConfig,
+    xa: Transactor[F]
+) {
   private[this] val logger = docspell.logging.getLogger[F]
 
   private def createLocations(folder: String) =
@@ -49,28 +53,46 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
   def run: F[MigrateResult] =
     for {
       _ <- runFixups
-      fw <- createFlyway(MigrationKind.Main)
+      result <- runMain
-      _ <- logger.info(s"!!! Running main migrations")
-      result <- Sync[F].blocking(fw.migrate())
     } yield result
 
+  def runMain: F[MigrateResult] =
+    if (!cfg.runMainMigrations)
+      logger
+        .info("Running main migrations is disabled!")
+        .as(new MigrateResult("", "", ""))
+    else
+      for {
+        fw <- createFlyway(MigrationKind.Main)
+        _ <- logger.info(s"!!! Running main migrations (repair=${cfg.repairSchema})")
+        _ <- if (cfg.repairSchema) Sync[F].blocking(fw.repair()).void else ().pure[F]
+        result <- Sync[F].blocking(fw.migrate())
+      } yield result
+
   // A hack to fix already published migrations
   def runFixups: F[Unit] =
-    isSchemaEmpty.flatMap {
+    if (!cfg.runFixupMigrations) logger.info(s"Running fixup migrations is disabled!")
-      case true =>
+    else
-        ().pure[F]
+      isSchemaEmpty.flatMap {
-      case false =>
+        case true =>
-        (for {
+          ().pure[F]
-          current <- OptionT(getSchemaVersion)
+        case false =>
-          _ <- OptionT
+          (for {
-            .fromOption[F](versionComponents(current))
+            current <- OptionT(getSchemaVersion)
-            .filter(v => v._1 >= 1 && v._2 >= 32)
+            _ <- OptionT
-          fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
+              .fromOption[F](versionComponents(current))
-          _ <- OptionT.liftF(logger.info(s"!!! Running fixup migrations"))
+              .filter(v => v._1 >= 1 && v._2 >= 32)
-          _ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
+            fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
-        } yield ())
+            _ <- OptionT.liftF(
-          .getOrElseF(logger.info(s"Fixup migrations not applied."))
+              logger.info(s"!!! Running fixup migrations (repair=${cfg.repairSchema})")
-    }
+            )
+            _ <-
+              if (cfg.repairSchema) OptionT.liftF(Sync[F].blocking(fw.repair()).void)
+              else OptionT.pure[F](())
+            _ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
+          } yield ())
+            .getOrElseF(logger.info(s"Fixup migrations not applied."))
+      }
 
   private def isSchemaEmpty: F[Boolean] =
     sql"select count(1) from flyway_schema_history"
@@ -95,8 +117,12 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
 }
 
 object FlywayMigrate {
-  def apply[F[_]: Sync](jdbcConfig: JdbcConfig, xa: Transactor[F]): FlywayMigrate[F] =
+  def apply[F[_]: Sync](
-    new FlywayMigrate[F](jdbcConfig, xa)
+      jdbcConfig: JdbcConfig,
+      schemaCfg: SchemaMigrateConfig,
+      xa: Transactor[F]
+  ): FlywayMigrate[F] =
+    new FlywayMigrate[F](jdbcConfig, schemaCfg, xa)
 
   sealed trait MigrationKind {
     def table: String

modules/store/src/test/scala/docspell/store/StoreFixture.scala

@@ -22,13 +22,15 @@ import org.mariadb.jdbc.MariaDbDataSource
 import org.postgresql.ds.PGConnectionPoolDataSource
 
 trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
+  def schemaMigrateConfig =
+    StoreFixture.schemaMigrateConfig
 
   val xa = ResourceFixture {
     val cfg = StoreFixture.memoryDB("test")
     for {
       ds <- StoreFixture.dataSource(cfg)
       xa <- StoreFixture.makeXA(ds)
-      _ <- Resource.eval(FlywayMigrate[IO](cfg, xa).run)
+      _ <- Resource.eval(FlywayMigrate[IO](cfg, schemaMigrateConfig, xa).run)
     } yield xa
   }
 
@@ -42,6 +44,7 @@ trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
 }
 
 object StoreFixture {
+  val schemaMigrateConfig = SchemaMigrateConfig.defaults
 
   def memoryDB(dbname: String): JdbcConfig =
     JdbcConfig(
@@ -94,7 +97,7 @@ object StoreFixture {
       xa <- makeXA(ds)
       cfg = FileRepositoryConfig.Database(64 * 1024)
       fr = FileRepository[IO](xa, ds, cfg, true)
-      store = new StoreImpl[IO](fr, jdbc, ds, xa)
+      store = new StoreImpl[IO](fr, jdbc, schemaMigrateConfig, ds, xa)
       _ <- Resource.eval(store.migrate)
     } yield store
 

modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala

@@ -10,7 +10,7 @@ import cats.effect.IO
 import cats.effect.unsafe.implicits._
 
 import docspell.logging.TestLoggingConfig
-import docspell.store.StoreFixture
+import docspell.store.{SchemaMigrateConfig, StoreFixture}
 
 import munit.FunSuite
 
@@ -21,7 +21,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
     val ds = StoreFixture.dataSource(jdbc)
     val result =
       ds.flatMap(StoreFixture.makeXA).use { xa =>
-        FlywayMigrate[IO](jdbc, xa).run
+        FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
       }
 
     assert(result.unsafeRunSync().migrationsExecuted > 0)
@@ -40,7 +40,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
 
     val result =
       ds.flatMap(StoreFixture.makeXA).use { xa =>
-        FlywayMigrate[IO](jdbc, xa).run
+        FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
       }
 
     result.unsafeRunSync()

modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala

@@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
 
 import docspell.common.LenientUri
 import docspell.logging.TestLoggingConfig
-import docspell.store.{JdbcConfig, StoreFixture}
+import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
 
 import com.dimafeng.testcontainers.MariaDBContainer
 import com.dimafeng.testcontainers.munit.TestContainerForAll
@@ -32,7 +32,7 @@ class MariaDbMigrateTest
         JdbcConfig(LenientUri.unsafe(cnt.jdbcUrl), cnt.dbUsername, cnt.dbPassword)
       val ds = StoreFixture.dataSource(jdbc)
       val result = ds.flatMap(StoreFixture.makeXA).use { xa =>
-        FlywayMigrate[IO](jdbc, xa).run
+        FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
       }
       assert(result.unsafeRunSync().migrationsExecuted > 0)
       // a second time to apply fixup migrations

modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala

@@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
 
 import docspell.common.LenientUri
 import docspell.logging.TestLoggingConfig
-import docspell.store.{JdbcConfig, StoreFixture}
+import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
 
 import com.dimafeng.testcontainers.PostgreSQLContainer
 import com.dimafeng.testcontainers.munit.TestContainerForAll
@@ -34,7 +34,7 @@ class PostgresqlMigrateTest
       val ds = StoreFixture.dataSource(jdbc)
       val result =
         ds.flatMap(StoreFixture.makeXA).use { xa =>
-          FlywayMigrate[IO](jdbc, xa).run
+          FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
         }
       assert(result.unsafeRunSync().migrationsExecuted > 0)