From 3764f9265b742641a4a3024677411d148f68c1f3 Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Sun, 22 May 2022 00:07:36 +0200
Subject: [PATCH] Configure run/repair db migrations
Refs: #1517
---
.../main/scala/docspell/backend/Config.scala | 3 +-
.../joex/src/main/resources/reference.conf | 14 ++++
.../src/main/scala/docspell/joex/Config.scala | 3 +-
.../main/scala/docspell/joex/JoexServer.scala | 1 +
.../src/main/resources/reference.conf | 14 ++++
.../docspell/restserver/RestServer.scala | 1 +
.../docspell/store/SchemaMigrateConfig.scala | 17 +++++
.../src/main/scala/docspell/store/Store.scala | 3 +-
.../scala/docspell/store/impl/StoreImpl.scala | 5 +-
.../store/migrate/FlywayMigrate.scala | 70 +++++++++++++------
.../scala/docspell/store/StoreFixture.scala | 7 +-
.../store/migrate/H2MigrateTest.scala | 6 +-
.../store/migrate/MariaDbMigrateTest.scala | 4 +-
.../store/migrate/PostgresqlMigrateTest.scala | 4 +-
14 files changed, 116 insertions(+), 36 deletions(-)
create mode 100644 modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala
diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala
index b1baedcb..03f0e23b 100644
--- a/modules/backend/src/main/scala/docspell/backend/Config.scala
+++ b/modules/backend/src/main/scala/docspell/backend/Config.scala
@@ -11,14 +11,15 @@ import cats.implicits._
import docspell.backend.signup.{Config => SignupConfig}
import docspell.common._
-import docspell.store.JdbcConfig
import docspell.store.file.FileRepositoryConfig
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
import emil.javamail.Settings
case class Config(
mailDebug: Boolean,
jdbc: JdbcConfig,
+ databaseSchema: SchemaMigrateConfig,
signup: SignupConfig,
files: Config.Files,
addons: Config.Addons
diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf
index a0c9e9b7..1cda608a 100644
--- a/modules/joex/src/main/resources/reference.conf
+++ b/modules/joex/src/main/resources/reference.conf
@@ -55,6 +55,20 @@ docspell.joex {
password = ""
}
+ # Additional settings related to schema migration.
+ database-schema = {
+ # Whether to run main database migrations.
+ run-main-migrations = true
+
+ # Whether to run the fixup migrations.
+ run-fixup-migrations = true
+
+ # Use with care. This repairs all migrations in the database by
+ # updating their checksums and removing failed migrations. Good
+ # for testing, not recommended for normal operation.
+ repair-schema = false
+ }
+
# Enable or disable debugging for e-mail related functionality. This
# applies to both sending and receiving mails. For security reasons
# logging is not very extensive on authentication failures. Setting
diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala
index 646033c2..4601f9a0 100644
--- a/modules/joex/src/main/scala/docspell/joex/Config.scala
+++ b/modules/joex/src/main/scala/docspell/joex/Config.scala
@@ -25,7 +25,7 @@ import docspell.joex.updatecheck.UpdateCheckConfig
import docspell.logging.LogConfig
import docspell.pubsub.naive.PubSubConfig
import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
-import docspell.store.JdbcConfig
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
case class Config(
appId: Ident,
@@ -33,6 +33,7 @@ case class Config(
logging: LogConfig,
bind: Config.Bind,
jdbc: JdbcConfig,
+ databaseSchema: SchemaMigrateConfig,
scheduler: SchedulerConfig,
periodicScheduler: PeriodicSchedulerConfig,
userTasks: Config.UserTasks,
diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala
index ae1bf1a7..a680844d 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala
@@ -41,6 +41,7 @@ object JoexServer {
store <- Store.create[F](
cfg.jdbc,
+ cfg.databaseSchema,
cfg.files.defaultFileRepositoryConfig,
pools.connectEC
)
diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf
index 2c6d4523..ccf076ad 100644
--- a/modules/restserver/src/main/resources/reference.conf
+++ b/modules/restserver/src/main/resources/reference.conf
@@ -387,6 +387,20 @@ docspell.server {
password = ""
}
+ # Additional settings related to schema migration.
+ database-schema = {
+ # Whether to run main database migrations.
+ run-main-migrations = true
+
+ # Whether to run the fixup migrations.
+ run-fixup-migrations = true
+
+ # Use with care. This repairs all migrations in the database by
+ # updating their checksums and removing failed migrations. Good
+ # for testing, not recommended for normal operation.
+ repair-schema = false
+ }
+
# Configuration for registering new users.
signup {
diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
index e4c1a5bc..4eb88c30 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
@@ -83,6 +83,7 @@ object RestServer {
httpClient <- BlazeClientBuilder[F].resource
store <- Store.create[F](
cfg.backend.jdbc,
+ cfg.backend.databaseSchema,
cfg.backend.files.defaultFileRepositoryConfig,
pools.connectEC
)
diff --git a/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala b/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala
new file mode 100644
index 00000000..8659ef99
--- /dev/null
+++ b/modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store
+
+case class SchemaMigrateConfig(
+ runMainMigrations: Boolean,
+ runFixupMigrations: Boolean,
+ repairSchema: Boolean
+)
+
+object SchemaMigrateConfig {
+ val defaults = SchemaMigrateConfig(true, true, false)
+}
diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala
index d41e83d2..f1c327c0 100644
--- a/modules/store/src/main/scala/docspell/store/Store.scala
+++ b/modules/store/src/main/scala/docspell/store/Store.scala
@@ -42,6 +42,7 @@ object Store {
def create[F[_]: Async](
jdbc: JdbcConfig,
+ schemaCfg: SchemaMigrateConfig,
fileRepoConfig: FileRepositoryConfig,
connectEC: ExecutionContext
): Resource[F, Store[F]] = {
@@ -58,7 +59,7 @@ object Store {
}
xa = HikariTransactor(ds, connectEC)
fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
- st = new StoreImpl[F](fr, jdbc, ds, xa)
+ st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa)
_ <- Resource.eval(st.migrate)
} yield st
}
diff --git a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala
index d68ef6e3..f49b1035 100644
--- a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala
+++ b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala
@@ -13,9 +13,9 @@ import cats.effect.Async
import cats.implicits._
import cats.~>
+import docspell.store._
import docspell.store.file.{FileRepository, FileRepositoryConfig}
import docspell.store.migrate.FlywayMigrate
-import docspell.store.{AddResult, JdbcConfig, Store}
import doobie._
import doobie.implicits._
@@ -23,6 +23,7 @@ import doobie.implicits._
final class StoreImpl[F[_]: Async](
val fileRepo: FileRepository[F],
jdbc: JdbcConfig,
+ schemaCfg: SchemaMigrateConfig,
ds: DataSource,
val transactor: Transactor[F]
) extends Store[F] {
@@ -38,7 +39,7 @@ final class StoreImpl[F[_]: Async](
FunctionK.lift(transact)
def migrate: F[Int] =
- FlywayMigrate[F](jdbc, xa).run.map(_.migrationsExecuted)
+ FlywayMigrate[F](jdbc, schemaCfg, xa).run.map(_.migrationsExecuted)
def transact[A](prg: ConnectionIO[A]): F[A] =
prg.transact(xa)
diff --git a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala
index 5df5ba1d..e0b1e2d5 100644
--- a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala
+++ b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala
@@ -10,15 +10,19 @@ import cats.data.OptionT
import cats.effect.Sync
import cats.implicits._
-import docspell.store.JdbcConfig
import docspell.store.migrate.FlywayMigrate.MigrationKind
+import docspell.store.{JdbcConfig, SchemaMigrateConfig}
import doobie.implicits._
import doobie.util.transactor.Transactor
import org.flywaydb.core.Flyway
import org.flywaydb.core.api.output.MigrateResult
-class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
+class FlywayMigrate[F[_]: Sync](
+ jdbc: JdbcConfig,
+ cfg: SchemaMigrateConfig,
+ xa: Transactor[F]
+) {
private[this] val logger = docspell.logging.getLogger[F]
private def createLocations(folder: String) =
@@ -49,28 +53,46 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
def run: F[MigrateResult] =
for {
_ <- runFixups
- fw <- createFlyway(MigrationKind.Main)
- _ <- logger.info(s"!!! Running main migrations")
- result <- Sync[F].blocking(fw.migrate())
+ result <- runMain
} yield result
+ def runMain: F[MigrateResult] =
+ if (!cfg.runMainMigrations)
+ logger
+ .info("Running main migrations is disabled!")
+ .as(new MigrateResult("", "", ""))
+ else
+ for {
+ fw <- createFlyway(MigrationKind.Main)
+ _ <- logger.info(s"!!! Running main migrations (repair=${cfg.repairSchema})")
+ _ <- if (cfg.repairSchema) Sync[F].blocking(fw.repair()).void else ().pure[F]
+ result <- Sync[F].blocking(fw.migrate())
+ } yield result
+
// A hack to fix already published migrations
def runFixups: F[Unit] =
- isSchemaEmpty.flatMap {
- case true =>
- ().pure[F]
- case false =>
- (for {
- current <- OptionT(getSchemaVersion)
- _ <- OptionT
- .fromOption[F](versionComponents(current))
- .filter(v => v._1 >= 1 && v._2 >= 32)
- fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
- _ <- OptionT.liftF(logger.info(s"!!! Running fixup migrations"))
- _ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
- } yield ())
- .getOrElseF(logger.info(s"Fixup migrations not applied."))
- }
+ if (!cfg.runFixupMigrations) logger.info(s"Running fixup migrations is disabled!")
+ else
+ isSchemaEmpty.flatMap {
+ case true =>
+ ().pure[F]
+ case false =>
+ (for {
+ current <- OptionT(getSchemaVersion)
+ _ <- OptionT
+ .fromOption[F](versionComponents(current))
+ .filter(v => v._1 >= 1 && v._2 >= 32)
+ fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
+ _ <- OptionT.liftF(
+ logger.info(s"!!! Running fixup migrations (repair=${cfg.repairSchema})")
+ )
+ _ <-
+ if (cfg.repairSchema) OptionT.liftF(Sync[F].blocking(fw.repair()).void)
+ else OptionT.pure[F](())
+ _ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
+ } yield ())
+ .getOrElseF(logger.info(s"Fixup migrations not applied."))
+ }
private def isSchemaEmpty: F[Boolean] =
sql"select count(1) from flyway_schema_history"
@@ -95,8 +117,12 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
}
object FlywayMigrate {
- def apply[F[_]: Sync](jdbcConfig: JdbcConfig, xa: Transactor[F]): FlywayMigrate[F] =
- new FlywayMigrate[F](jdbcConfig, xa)
+ def apply[F[_]: Sync](
+ jdbcConfig: JdbcConfig,
+ schemaCfg: SchemaMigrateConfig,
+ xa: Transactor[F]
+ ): FlywayMigrate[F] =
+ new FlywayMigrate[F](jdbcConfig, schemaCfg, xa)
sealed trait MigrationKind {
def table: String
diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala
index eca434ce..cbff6a1f 100644
--- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala
+++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala
@@ -22,13 +22,15 @@ import org.mariadb.jdbc.MariaDbDataSource
import org.postgresql.ds.PGConnectionPoolDataSource
trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
+ def schemaMigrateConfig =
+ StoreFixture.schemaMigrateConfig
val xa = ResourceFixture {
val cfg = StoreFixture.memoryDB("test")
for {
ds <- StoreFixture.dataSource(cfg)
xa <- StoreFixture.makeXA(ds)
- _ <- Resource.eval(FlywayMigrate[IO](cfg, xa).run)
+ _ <- Resource.eval(FlywayMigrate[IO](cfg, schemaMigrateConfig, xa).run)
} yield xa
}
@@ -42,6 +44,7 @@ trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
}
object StoreFixture {
+ val schemaMigrateConfig = SchemaMigrateConfig.defaults
def memoryDB(dbname: String): JdbcConfig =
JdbcConfig(
@@ -94,7 +97,7 @@ object StoreFixture {
xa <- makeXA(ds)
cfg = FileRepositoryConfig.Database(64 * 1024)
fr = FileRepository[IO](xa, ds, cfg, true)
- store = new StoreImpl[IO](fr, jdbc, ds, xa)
+ store = new StoreImpl[IO](fr, jdbc, schemaMigrateConfig, ds, xa)
_ <- Resource.eval(store.migrate)
} yield store
diff --git a/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala
index 4e47d5c2..cb51023a 100644
--- a/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala
+++ b/modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala
@@ -10,7 +10,7 @@ import cats.effect.IO
import cats.effect.unsafe.implicits._
import docspell.logging.TestLoggingConfig
-import docspell.store.StoreFixture
+import docspell.store.{SchemaMigrateConfig, StoreFixture}
import munit.FunSuite
@@ -21,7 +21,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
val ds = StoreFixture.dataSource(jdbc)
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
- FlywayMigrate[IO](jdbc, xa).run
+ FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)
@@ -40,7 +40,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
- FlywayMigrate[IO](jdbc, xa).run
+ FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
result.unsafeRunSync()
diff --git a/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala
index a77382dc..75e765b6 100644
--- a/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala
+++ b/modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala
@@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
import docspell.common.LenientUri
import docspell.logging.TestLoggingConfig
-import docspell.store.{JdbcConfig, StoreFixture}
+import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
import com.dimafeng.testcontainers.MariaDBContainer
import com.dimafeng.testcontainers.munit.TestContainerForAll
@@ -32,7 +32,7 @@ class MariaDbMigrateTest
JdbcConfig(LenientUri.unsafe(cnt.jdbcUrl), cnt.dbUsername, cnt.dbPassword)
val ds = StoreFixture.dataSource(jdbc)
val result = ds.flatMap(StoreFixture.makeXA).use { xa =>
- FlywayMigrate[IO](jdbc, xa).run
+ FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)
// a second time to apply fixup migrations
diff --git a/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala b/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala
index 235b240e..f71d01c8 100644
--- a/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala
+++ b/modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala
@@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
import docspell.common.LenientUri
import docspell.logging.TestLoggingConfig
-import docspell.store.{JdbcConfig, StoreFixture}
+import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
import com.dimafeng.testcontainers.PostgreSQLContainer
import com.dimafeng.testcontainers.munit.TestContainerForAll
@@ -34,7 +34,7 @@ class PostgresqlMigrateTest
val ds = StoreFixture.dataSource(jdbc)
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
- FlywayMigrate[IO](jdbc, xa).run
+ FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)