TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-04 10:29:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Configure run/repair db migrations

Browse Source 
Refs: #1517
This commit is contained in:
eikek 2022-05-22 00:07:36 +02:00
parent 79688c7711
commit 3764f9265b
14 changed files with 116 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/backend/src/main/scala/docspell/backend/Config.scala
View File

@ -11,14 +11,15 @@ import cats.implicits._
import docspell.backend.signup.{Config => SignupConfig}
import docspell.common._
import docspell.store.JdbcConfig
import docspell.store.file.FileRepositoryConfig
import docspell.store.{JdbcConfig, SchemaMigrateConfig}
import emil.javamail.Settings
case class Config(
mailDebug: Boolean,
jdbc: JdbcConfig,
databaseSchema: SchemaMigrateConfig,
signup: SignupConfig,
files: Config.Files,
addons: Config.Addons

14
modules/joex/src/main/resources/reference.conf
View File

@ -55,6 +55,20 @@ docspell.joex {
password = ""
}
# Additional settings related to schema migration.
database-schema = {
# Whether to run main database migrations.
run-main-migrations = true
# Whether to run the fixup migrations.
run-fixup-migrations = true
# Use with care. This repairs all migrations in the database by
# updating their checksums and removing failed migrations. Good
# for testing, not recommended for normal operation.
repair-schema = false
}
# Enable or disable debugging for e-mail related functionality. This
# applies to both sending and receiving mails. For security reasons
# logging is not very extensive on authentication failures. Setting

3
modules/joex/src/main/scala/docspell/joex/Config.scala
View File

@ -25,7 +25,7 @@ import docspell.joex.updatecheck.UpdateCheckConfig
import docspell.logging.LogConfig
import docspell.pubsub.naive.PubSubConfig
import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
import docspell.store.JdbcConfig
import docspell.store.{JdbcConfig, SchemaMigrateConfig}
case class Config(
appId: Ident,
@ -33,6 +33,7 @@ case class Config(
logging: LogConfig,
bind: Config.Bind,
jdbc: JdbcConfig,
databaseSchema: SchemaMigrateConfig,
scheduler: SchedulerConfig,
periodicScheduler: PeriodicSchedulerConfig,
userTasks: Config.UserTasks,

1
modules/joex/src/main/scala/docspell/joex/JoexServer.scala
View File

@ -41,6 +41,7 @@ object JoexServer {
store <- Store.create[F](
cfg.jdbc,
cfg.databaseSchema,
cfg.files.defaultFileRepositoryConfig,
pools.connectEC
)

14
modules/restserver/src/main/resources/reference.conf
View File

@ -387,6 +387,20 @@ docspell.server {
password = ""
}
# Additional settings related to schema migration.
database-schema = {
# Whether to run main database migrations.
run-main-migrations = true
# Whether to run the fixup migrations.
run-fixup-migrations = true
# Use with care. This repairs all migrations in the database by
# updating their checksums and removing failed migrations. Good
# for testing, not recommended for normal operation.
repair-schema = false
}
# Configuration for registering new users.
signup {

1
modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
View File

@ -83,6 +83,7 @@ object RestServer {
httpClient <- BlazeClientBuilder[F].resource
store <- Store.create[F](
cfg.backend.jdbc,
cfg.backend.databaseSchema,
cfg.backend.files.defaultFileRepositoryConfig,
pools.connectEC
)

17
modules/store/src/main/scala/docspell/store/SchemaMigrateConfig.scala Normal file
View File

@ -0,0 +1,17 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.store
case class SchemaMigrateConfig(
runMainMigrations: Boolean,
runFixupMigrations: Boolean,
repairSchema: Boolean
)
object SchemaMigrateConfig {
val defaults = SchemaMigrateConfig(true, true, false)
}

3
modules/store/src/main/scala/docspell/store/Store.scala
View File

@ -42,6 +42,7 @@ object Store {
def create[F[_]: Async](
jdbc: JdbcConfig,
schemaCfg: SchemaMigrateConfig,
fileRepoConfig: FileRepositoryConfig,
connectEC: ExecutionContext
): Resource[F, Store[F]] = {
@ -58,7 +59,7 @@ object Store {
}
xa = HikariTransactor(ds, connectEC)
fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
st = new StoreImpl[F](fr, jdbc, ds, xa)
st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa)
_ <- Resource.eval(st.migrate)
} yield st
}

5
modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala
View File

@ -13,9 +13,9 @@ import cats.effect.Async
import cats.implicits._
import cats.~>
import docspell.store._
import docspell.store.file.{FileRepository, FileRepositoryConfig}
import docspell.store.migrate.FlywayMigrate
import docspell.store.{AddResult, JdbcConfig, Store}
import doobie._
import doobie.implicits._
@ -23,6 +23,7 @@ import doobie.implicits._
final class StoreImpl[F[_]: Async](
val fileRepo: FileRepository[F],
jdbc: JdbcConfig,
schemaCfg: SchemaMigrateConfig,
ds: DataSource,
val transactor: Transactor[F]
) extends Store[F] {
@ -38,7 +39,7 @@ final class StoreImpl[F[_]: Async](
FunctionK.lift(transact)
def migrate: F[Int] =
FlywayMigrate[F](jdbc, xa).run.map(_.migrationsExecuted)
FlywayMigrate[F](jdbc, schemaCfg, xa).run.map(_.migrationsExecuted)
def transact[A](prg: ConnectionIO[A]): F[A] =
prg.transact(xa)

70
modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala
View File

@ -10,15 +10,19 @@ import cats.data.OptionT
import cats.effect.Sync
import cats.implicits._
import docspell.store.JdbcConfig
import docspell.store.migrate.FlywayMigrate.MigrationKind
import docspell.store.{JdbcConfig, SchemaMigrateConfig}
import doobie.implicits._
import doobie.util.transactor.Transactor
import org.flywaydb.core.Flyway
import org.flywaydb.core.api.output.MigrateResult
class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
class FlywayMigrate[F[_]: Sync](
jdbc: JdbcConfig,
cfg: SchemaMigrateConfig,
xa: Transactor[F]
) {
private[this] val logger = docspell.logging.getLogger[F]
private def createLocations(folder: String) =
@ -49,28 +53,46 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
def run: F[MigrateResult] =
for {
_ <- runFixups
fw <- createFlyway(MigrationKind.Main)
_ <- logger.info(s"!!! Running main migrations")
result <- Sync[F].blocking(fw.migrate())
result <- runMain
} yield result
def runMain: F[MigrateResult] =
if (!cfg.runMainMigrations)
logger
.info("Running main migrations is disabled!")
.as(new MigrateResult("", "", ""))
else
for {
fw <- createFlyway(MigrationKind.Main)
_ <- logger.info(s"!!! Running main migrations (repair=${cfg.repairSchema})")
_ <- if (cfg.repairSchema) Sync[F].blocking(fw.repair()).void else ().pure[F]
result <- Sync[F].blocking(fw.migrate())
} yield result
// A hack to fix already published migrations
def runFixups: F[Unit] =
isSchemaEmpty.flatMap {
case true =>
().pure[F]
case false =>
(for {
current <- OptionT(getSchemaVersion)
_ <- OptionT
.fromOption[F](versionComponents(current))
.filter(v => v._1 >= 1 && v._2 >= 32)
fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
_ <- OptionT.liftF(logger.info(s"!!! Running fixup migrations"))
_ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
} yield ())
.getOrElseF(logger.info(s"Fixup migrations not applied."))
}
if (!cfg.runFixupMigrations) logger.info(s"Running fixup migrations is disabled!")
else
isSchemaEmpty.flatMap {
case true =>
().pure[F]
case false =>
(for {
current <- OptionT(getSchemaVersion)
_ <- OptionT
.fromOption[F](versionComponents(current))
.filter(v => v._1 >= 1 && v._2 >= 32)
fw <- OptionT.liftF(createFlyway(MigrationKind.Fixups))
_ <- OptionT.liftF(
logger.info(s"!!! Running fixup migrations (repair=${cfg.repairSchema})")
)
_ <-
if (cfg.repairSchema) OptionT.liftF(Sync[F].blocking(fw.repair()).void)
else OptionT.pure[F](())
_ <- OptionT.liftF(Sync[F].blocking(fw.migrate()))
} yield ())
.getOrElseF(logger.info(s"Fixup migrations not applied."))
}
private def isSchemaEmpty: F[Boolean] =
sql"select count(1) from flyway_schema_history"
@ -95,8 +117,12 @@ class FlywayMigrate[F[_]: Sync](jdbc: JdbcConfig, xa: Transactor[F]) {
}
object FlywayMigrate {
def apply[F[_]: Sync](jdbcConfig: JdbcConfig, xa: Transactor[F]): FlywayMigrate[F] =
new FlywayMigrate[F](jdbcConfig, xa)
def apply[F[_]: Sync](
jdbcConfig: JdbcConfig,
schemaCfg: SchemaMigrateConfig,
xa: Transactor[F]
): FlywayMigrate[F] =
new FlywayMigrate[F](jdbcConfig, schemaCfg, xa)
sealed trait MigrationKind {
def table: String

7
modules/store/src/test/scala/docspell/store/StoreFixture.scala
View File

@ -22,13 +22,15 @@ import org.mariadb.jdbc.MariaDbDataSource
import org.postgresql.ds.PGConnectionPoolDataSource
trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
def schemaMigrateConfig =
StoreFixture.schemaMigrateConfig
val xa = ResourceFixture {
val cfg = StoreFixture.memoryDB("test")
for {
ds <- StoreFixture.dataSource(cfg)
xa <- StoreFixture.makeXA(ds)
_ <- Resource.eval(FlywayMigrate[IO](cfg, xa).run)
_ <- Resource.eval(FlywayMigrate[IO](cfg, schemaMigrateConfig, xa).run)
} yield xa
}
@ -42,6 +44,7 @@ trait StoreFixture extends CatsEffectFunFixtures { self: CatsEffectSuite =>
}
object StoreFixture {
val schemaMigrateConfig = SchemaMigrateConfig.defaults
def memoryDB(dbname: String): JdbcConfig =
JdbcConfig(
@ -94,7 +97,7 @@ object StoreFixture {
xa <- makeXA(ds)
cfg = FileRepositoryConfig.Database(64 * 1024)
fr = FileRepository[IO](xa, ds, cfg, true)
store = new StoreImpl[IO](fr, jdbc, ds, xa)
store = new StoreImpl[IO](fr, jdbc, schemaMigrateConfig, ds, xa)
_ <- Resource.eval(store.migrate)
} yield store

6
modules/store/src/test/scala/docspell/store/migrate/H2MigrateTest.scala
View File

@ -10,7 +10,7 @@ import cats.effect.IO
import cats.effect.unsafe.implicits._
import docspell.logging.TestLoggingConfig
import docspell.store.StoreFixture
import docspell.store.{SchemaMigrateConfig, StoreFixture}
import munit.FunSuite
@ -21,7 +21,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
val ds = StoreFixture.dataSource(jdbc)
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
FlywayMigrate[IO](jdbc, xa).run
FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)
@ -40,7 +40,7 @@ class H2MigrateTest extends FunSuite with TestLoggingConfig {
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
FlywayMigrate[IO](jdbc, xa).run
FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
result.unsafeRunSync()

4
modules/store/src/test/scala/docspell/store/migrate/MariaDbMigrateTest.scala
View File

@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
import docspell.common.LenientUri
import docspell.logging.TestLoggingConfig
import docspell.store.{JdbcConfig, StoreFixture}
import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
import com.dimafeng.testcontainers.MariaDBContainer
import com.dimafeng.testcontainers.munit.TestContainerForAll
@ -32,7 +32,7 @@ class MariaDbMigrateTest
JdbcConfig(LenientUri.unsafe(cnt.jdbcUrl), cnt.dbUsername, cnt.dbPassword)
val ds = StoreFixture.dataSource(jdbc)
val result = ds.flatMap(StoreFixture.makeXA).use { xa =>
FlywayMigrate[IO](jdbc, xa).run
FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)
// a second time to apply fixup migrations

4
modules/store/src/test/scala/docspell/store/migrate/PostgresqlMigrateTest.scala
View File

@ -11,7 +11,7 @@ import cats.effect.unsafe.implicits._
import docspell.common.LenientUri
import docspell.logging.TestLoggingConfig
import docspell.store.{JdbcConfig, StoreFixture}
import docspell.store.{JdbcConfig, SchemaMigrateConfig, StoreFixture}
import com.dimafeng.testcontainers.PostgreSQLContainer
import com.dimafeng.testcontainers.munit.TestContainerForAll
@ -34,7 +34,7 @@ class PostgresqlMigrateTest
val ds = StoreFixture.dataSource(jdbc)
val result =
ds.flatMap(StoreFixture.makeXA).use { xa =>
FlywayMigrate[IO](jdbc, xa).run
FlywayMigrate[IO](jdbc, SchemaMigrateConfig.defaults, xa).run
}
assert(result.unsafeRunSync().migrationsExecuted > 0)