Merge pull request #1690 from eikek/h2-update

Update h2 and flyway

docker/dockerfiles/joex.dockerfile

@@ -53,7 +53,7 @@ RUN JDKPKG="openjdk11-jre"; \
     qpdf-dev \
     openssl-dev \
     ocrmypdf \
-  && apk add 'zlib=1.2.12-r2' \
+  && apk add 'zlib=1.2.12-r3' \
   && pip3 install --upgrade pip \
   && pip3 install ocrmypdf \
   && curl -Ls $UNO_URL -o /usr/local/bin/unoconv \

docker/dockerfiles/restserver.dockerfile

@@ -8,7 +8,7 @@ RUN JDKPKG="openjdk11-jre"; \
     if [[ $TARGETPLATFORM = linux/arm* ]]; then JDKPKG="openjdk8-jre"; fi; \
     apk update && \
     apk add --no-cache $JDKPKG bash tzdata && \
-    apk add 'zlib=1.2.12-r2'
+    apk add 'zlib=1.2.12-r3'
 
 WORKDIR /opt
 RUN wget ${restserver_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-restserver-$version.zip} && \

modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QJob.scala

@@ -89,7 +89,7 @@ object QJob {
       res <- job.traverse(j => markJob(j))
     } yield res.map(_.map(_.some)).getOrElse {
       if (group.isDefined)
-        Left(()) // if a group was found, but no job someone else was faster
+        Left(()) // if a group was found but no job, someone else was faster
       else Right(None)
     }
   }
@@ -115,33 +115,27 @@ object QJob {
       val selectAll = Select(JC.group.s, from(JC), stateCond).distinct
     }
 
-    val sql1 =
-      Select(
-        select(min(AllGroups.group).as("g"), lit("0 as n")),
-        from(AllGroups),
-        AllGroups.group > Select(G.group.s, from(G), G.worker === worker)
-      )
-
-    val sql2 =
-      Select(
-        select(min(AllGroups.group).as("g"), lit("1 as n")),
-        from(AllGroups)
-      )
-
-    val gcol = Column[String]("g", TableDef(""))
-    val gnum = Column[Int]("n", TableDef(""))
     val groups =
       withCte(AllGroups -> AllGroups.selectAll)
-        .select(Select(gcol.s, from(union(sql1, sql2), "t0"), gcol.isNull.negate))
-        .orderBy(gnum.asc)
-        .limit(1)
-
-    val frag = groups.build
-    cioLogger.trace(
-      s"nextGroupQuery: $frag  (now=${now.toMillis}, pause=${initialPause.millis})"
+        .select(
+          Select(
+            coalesce(
+              Select(
+                select(min(AllGroups.group)),
+                from(AllGroups),
+                AllGroups.group > Select(G.group.s, from(G), G.worker === worker)
+              ).asSubSelect,
+              Select(select(min(AllGroups.group)), from(AllGroups)).asSubSelect
+            ).s
+          )
         )
 
-    frag.query[Ident].option
+    val frag = groups.build
+    cioLogger
+      .trace(
+        s"nextGroupQuery: $frag  (now=${now.toMillis}, pause=${initialPause.millis})"
+      ) *>
+      groups.build.query[Ident].option
   }
 
   private def stuckTriggerValue(t: RJob.Table, initialPause: Duration, now: Timestamp) =

modules/scheduler/impl/src/test/scala/docspell/scheduler/impl/QJobTest.scala

@@ -9,17 +9,13 @@ package docspell.scheduler.impl
 import java.time.Instant
 import java.util.concurrent.atomic.AtomicLong
 
-import cats.implicits._
+import cats.syntax.all._
 
 import docspell.common._
-import docspell.logging.TestLoggingConfig
-import docspell.store.StoreFixture
 import docspell.store.records.{RJob, RJobGroupUse}
+import docspell.store.{DatabaseTest, Db}
 
-import doobie.implicits._
-import munit._
-
-class QJobTest extends CatsEffectSuite with StoreFixture with TestLoggingConfig {
+class QJobTest extends DatabaseTest {
   private[this] val c = new AtomicLong(0)
 
   private val worker = Ident.unsafe("joex1")
@@ -28,6 +24,11 @@ class QJobTest extends CatsEffectSuite with StoreFixture with TestLoggingConfig
   private val group1 = Ident.unsafe("group1")
   private val group2 = Ident.unsafe("group2")
 
+  override def munitFixtures = h2File ++ mariaDbAll ++ postgresAll
+
+  def createStore(dbms: Db) =
+    dbms.fold(pgStore(), mariaStore(), h2FileStore())
+
   def createJob(group: Ident): RJob =
     RJob.fromJson[Unit](
       Ident.unsafe(s"job-${c.incrementAndGet()}"),
@@ -41,54 +42,66 @@ class QJobTest extends CatsEffectSuite with StoreFixture with TestLoggingConfig
       None
     )
 
-  xa.test("set group must insert or update") { tx =>
+  Db.all.toList.foreach { db =>
+    test(s"set group must insert or update ($db)") {
+      val store = createStore(db)
       val res =
         for {
-        _ <- RJobGroupUse.setGroup(RJobGroupUse(group1, worker)).transact(tx)
-        res <- RJobGroupUse.findGroup(worker).transact(tx)
+          _ <- store.transact(RJobGroupUse.setGroup(RJobGroupUse(group1, worker)))
+          res <- store.transact(RJobGroupUse.findGroup(worker))
         } yield res
 
       res.assertEquals(Some(group1))
     }
+  }
 
-  xa.test("selectNextGroup should return first group on initial state") { tx =>
+  Db.all.toList.foreach { db =>
+    test(s"selectNextGroup should return first group on initial state ($db)") {
+      val store = createStore(db)
       val nextGroup = for {
         _ <- List(group1, group2, group1, group2, group2)
           .map(createJob)
           .map(RJob.insert)
-        .traverse(_.transact(tx))
-      _ <- RJobGroupUse.deleteAll.transact(tx)
-      next <- QJob.selectNextGroup(worker, nowTs, initialPause).transact(tx)
+          .traverse_(store.transact(_))
+        _ <- store.transact(RJobGroupUse.deleteAll)
+        next <- store.transact(QJob.selectNextGroup(worker, nowTs, initialPause))
       } yield next
 
       nextGroup.assertEquals(Some(group1))
     }
+  }
 
-  xa.test("selectNextGroup should return second group on subsequent call (1)") { tx =>
+  Db.all.toList.foreach { db =>
+    test(s"selectNextGroup should return second group on subsequent call ($db)") {
+      val store = createStore(db)
       val nextGroup = for {
         _ <- List(group1, group2, group1, group2)
           .map(createJob)
           .map(RJob.insert)
-        .traverse(_.transact(tx))
-      _ <- RJobGroupUse.deleteAll.transact(tx)
-      _ <- RJobGroupUse.setGroup(RJobGroupUse(group1, worker)).transact(tx)
-      next <- QJob.selectNextGroup(worker, nowTs, initialPause).transact(tx)
+          .traverse_(store.transact(_))
+        _ <- store.transact(RJobGroupUse.deleteAll)
+        _ <- store.transact(RJobGroupUse.setGroup(RJobGroupUse(group1, worker)))
+        next <- store.transact(QJob.selectNextGroup(worker, nowTs, initialPause))
       } yield next
 
       nextGroup.assertEquals(Some(group2))
     }
+  }
 
-  xa.test("selectNextGroup should return second group on subsequent call (2)") { tx =>
+  Db.all.toList.foreach { db =>
+    test(s"selectNextGroup should return first group on subsequent call ($db)") {
+      val store = createStore(db)
       val nextGroup = for {
         _ <- List(group1, group2, group1, group2)
           .map(createJob)
           .map(RJob.insert)
-        .traverse(_.transact(tx))
-      _ <- RJobGroupUse.deleteAll.transact(tx)
-      _ <- RJobGroupUse.setGroup(RJobGroupUse(group2, worker)).transact(tx)
-      next <- QJob.selectNextGroup(worker, nowTs, initialPause).transact(tx)
+          .traverse_(store.transact(_))
+        _ <- store.transact(RJobGroupUse.deleteAll)
+        _ <- store.transact(RJobGroupUse.setGroup(RJobGroupUse(group2, worker)))
+        next <- store.transact(QJob.selectNextGroup(worker, nowTs, initialPause))
       } yield next
 
       nextGroup.assertEquals(Some(group1))
     }
+  }
 }

modules/store/src/main/resources/db/migration/h2/V1.39.0__collective_id.sql

@@ -1,5 +1,5 @@
 -- add new id column
-alter table "collective" add column "id" bigserial not null;
+alter table "collective" add column "id" bigserial not null unique;
 create unique index "collective_id_idx" on "collective"("id");
 
 -- change references: source

modules/store/src/main/resources/db/migration/h2/V1.39.3__rename_value.sql

@@ -0,0 +1 @@
+alter table "contact" rename column "value" to "value_";

modules/store/src/main/resources/db/migration/h2/V1.9.3__joblog_counter.sql

@@ -1,2 +1,2 @@
 ALTER TABLE "joblog"
-ADD COLUMN "counter" bigint auto_increment;
+ADD COLUMN "counter" bigint generated always as identity;

modules/store/src/main/resources/db/migration/mariadb/V1.39.3__rename_value.sql

@@ -0,0 +1 @@
+alter table `contact` rename column `value` to `value_`;

modules/store/src/main/resources/db/migration/postgresql/V1.39.3__rename_value.sql

@@ -0,0 +1 @@
+alter table "contact" rename column "value" to "value_";

modules/store/src/main/scala/docspell/store/qb/Select.scala

@@ -7,6 +7,7 @@
 package docspell.store.qb
 
 import cats.data.{NonEmptyList => Nel}
+import cats.syntax.option._
 
 import docspell.store.qb.impl.SelectBuilder
 
@@ -23,6 +24,9 @@ sealed trait Select {
   def as(alias: String): SelectExpr.SelectQuery =
     SelectExpr.SelectQuery(this, Some(alias))
 
+  def asSubSelect: SelectExpr.SelectQuery =
+    SelectExpr.SelectQuery(this, None)
+
   /** Adds one or more order-by definitions */
   def orderBy(ob: OrderBy, obs: OrderBy*): Select
 
@@ -71,35 +75,38 @@ sealed trait Select {
 }
 
 object Select {
+  def apply(projection: SelectExpr) =
+    SimpleSelect(false, Nel.of(projection), None, Condition.unit, None)
+
   def apply(projection: Nel[SelectExpr], from: FromExpr) =
-    SimpleSelect(false, projection, from, Condition.unit, None)
+    SimpleSelect(false, projection, from.some, Condition.unit, None)
 
   def apply(projection: SelectExpr, from: FromExpr) =
-    SimpleSelect(false, Nel.of(projection), from, Condition.unit, None)
+    SimpleSelect(false, Nel.of(projection), from.some, Condition.unit, None)
 
   def apply(
       projection: Nel[SelectExpr],
       from: FromExpr,
       where: Condition
-  ) = SimpleSelect(false, projection, from, where, None)
+  ) = SimpleSelect(false, projection, from.some, where, None)
 
   def apply(
       projection: SelectExpr,
       from: FromExpr,
       where: Condition
-  ) = SimpleSelect(false, Nel.of(projection), from, where, None)
+  ) = SimpleSelect(false, Nel.of(projection), from.some, where, None)
 
   def apply(
       projection: Nel[SelectExpr],
       from: FromExpr,
       where: Condition,
       groupBy: GroupBy
-  ) = SimpleSelect(false, projection, from, where, Some(groupBy))
+  ) = SimpleSelect(false, projection, from.some, where, Some(groupBy))
 
   case class SimpleSelect(
       distinctFlag: Boolean,
       projection: Nel[SelectExpr],
-      from: FromExpr,
+      from: Option[FromExpr],
       where: Condition,
       groupBy: Option[GroupBy]
   ) extends Select {
@@ -125,7 +132,7 @@ object Select {
       copy(projection = es)
 
     def changeFrom(f: FromExpr => FromExpr): SimpleSelect =
-      copy(from = f(from))
+      copy(from = from.map(f))
 
     def changeWhere(f: Condition => Condition): SimpleSelect =
       copy(where = f(where))

modules/store/src/main/scala/docspell/store/qb/SelectExpr.scala

@@ -33,6 +33,8 @@ object SelectExpr {
   case class SelectLiteral(value: String, alias: Option[String]) extends SelectExpr {
     def as(a: String): SelectLiteral =
       copy(alias = Some(a))
+    def as(otherCol: Column[_]): SelectExpr =
+      copy(alias = Some(otherCol.name))
   }
 
   case class SelectQuery(query: Select, alias: Option[String]) extends SelectExpr {

modules/store/src/main/scala/docspell/store/qb/impl/SelectBuilder.scala

@@ -52,7 +52,7 @@ object SelectBuilder {
 
   def buildSimple(sq: Select.SimpleSelect): Fragment = {
     val f0 = sq.projection.map(selectExpr).reduceLeft(_ ++ comma ++ _)
-    val f1 = fromExpr(sq.from)
+    val f1 = sq.from.map(fromExpr).getOrElse(Fragment.empty)
     val f2 = cond(sq.where)
     val f3 = sq.groupBy.map(groupBy).getOrElse(Fragment.empty)
     f0 ++ f1 ++ f2 ++ f3

modules/store/src/main/scala/docspell/store/records/RContact.scala

@@ -30,7 +30,7 @@ object RContact {
     val tableName = "contact"
 
     val contactId = Column[Ident]("contactid", this)
-    val value = Column[String]("value", this)
+    val value = Column[String]("value_", this)
     val kind = Column[ContactKind]("kind", this)
     val personId = Column[Ident]("pid", this)
     val orgId = Column[Ident]("oid", this)

modules/store/src/main/scala/docspell/store/records/RJob.scala

@@ -300,11 +300,6 @@ object RJob {
   def setProgress(jobId: Ident, perc: Int): ConnectionIO[Int] =
     DML.update(T, T.id === jobId, DML.set(T.progress.setTo(perc)))
 
-  def selectWaiting: ConnectionIO[Option[RJob]] = {
-    val sql = run(select(T.all), from(T), T.state === JobState.waiting)
-    sql.query[RJob].to[Vector].map(_.headOption)
-  }
-
   def selectGroupInState(states: NonEmptyList[JobState]): ConnectionIO[Vector[Ident]] = {
     val sql =
       Select(select(T.group), from(T), T.state.in(states)).orderBy(T.group)

modules/store/src/test/scala/docspell/store/DatabaseTest.scala

@@ -9,6 +9,8 @@ package docspell.store
 import java.util.UUID
 
 import cats.effect._
+import cats.syntax.option._
+import fs2.io.file.{Files, Path}
 
 import docspell.common._
 import docspell.logging.TestLoggingConfig
@@ -55,6 +57,15 @@ trait DatabaseTest
     }
   )
 
+  lazy val h2FileDataSource = ResourceSuiteLocalFixture(
+    "h2FileDataSource",
+    for {
+      file <- Files[IO].tempFile(Path("target").some, "h2-test-", ".db", None)
+      jdbc = StoreFixture.fileDB(file)
+      res <- StoreFixture.dataSource(jdbc).map(ds => (jdbc, ds))
+    } yield res
+  )
+
   lazy val newH2DataSource = ResourceFixture(for {
     jdbc <- Resource.eval(IO(StoreFixture.memoryDB(UUID.randomUUID().toString)))
     ds <- StoreFixture.dataSource(jdbc)
@@ -84,9 +95,18 @@ trait DatabaseTest
     } yield store
   )
 
+  lazy val h2FileStore = ResourceSuiteLocalFixture(
+    "h2FileStore",
+    for {
+      t <- Resource.eval(IO(h2FileDataSource()))
+      store <- StoreFixture.store(t._2, t._1)
+    } yield store
+  )
+
   def postgresAll = List(postgresCnt, pgDataSource, pgStore)
   def mariaDbAll = List(mariadbCnt, mariaDataSource, mariaStore)
-  def h2All = List(h2DataSource, h2Store)
+  def h2Memory = List(h2DataSource, h2Store)
+  def h2File = List(h2FileDataSource, h2FileStore)
 }
 
 object DatabaseTest {

modules/store/src/test/scala/docspell/store/StoreFixture.scala

@@ -9,6 +9,7 @@ package docspell.store
 import javax.sql.DataSource
 
 import cats.effect._
+import fs2.io.file.Path
 
 import docspell.common.LenientUri
 import docspell.store.file.{FileRepository, FileRepositoryConfig}
@@ -55,6 +56,15 @@ object StoreFixture {
       ""
     )
 
+  def fileDB(file: Path): JdbcConfig =
+    JdbcConfig(
+      LenientUri.unsafe(
+        s"jdbc:h2:file://${file.absolute.toString};MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"
+      ),
+      "sa",
+      ""
+    )
+
   def dataSource(jdbc: JdbcConfig): Resource[IO, JdbcConnectionPool] = {
     def jdbcConnPool =
       jdbc.dbms match {
@@ -115,5 +125,4 @@ object StoreFixture {
       case None =>
         IO.raiseError(new Exception(s"Resource not found: $resourceName"))
     }
-
 }

modules/store/src/test/scala/docspell/store/fts/TempFtsOpsTest.scala

@@ -27,7 +27,7 @@ import doobie._
 class TempFtsOpsTest extends DatabaseTest {
   private[this] val logger = docspell.logging.getLogger[IO]
 
-  override def munitFixtures = postgresAll ++ mariaDbAll ++ h2All
+  override def munitFixtures = postgresAll ++ mariaDbAll ++ h2Memory
 
   def id(str: String): Ident = Ident.unsafe(str)
 

modules/store/src/test/scala/docspell/store/qb/QueryBuilderTest.scala

@@ -50,9 +50,11 @@ class QueryBuilderTest extends FunSuite with TestLoggingConfig {
         )
         assertEquals(11, proj.size)
         from match {
-          case FromExpr.From(_) =>
+          case None =>
             fail("Unexpected from value")
-          case FromExpr.Joined(f, joins) =>
+          case Some(FromExpr.From(_)) =>
+            fail("Unexpected from value")
+          case Some(FromExpr.Joined(f, joins)) =>
             assertEquals(f, FromExpr.From(c))
             assertEquals(2, joins.size)
             joins.head match {

project/Dependencies.scala

@@ -19,9 +19,9 @@ object Dependencies {
   val DoobieVersion = "1.0.0-RC2"
   val EmilVersion = "0.12.0"
   val FlexmarkVersion = "0.64.0"
-  val FlywayVersion = "8.5.13"
+  val FlywayVersion = "9.1.3"
   val Fs2Version = "3.2.12"
-  val H2Version = "1.4.200"
+  val H2Version = "2.1.214"
   val Http4sVersion = "0.23.14"
   val Icu4jVersion = "71.1"
   val JavaOtpVersion = "0.4.0"

tools/h2-util.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+#
+# Create and restore dumps from h2 databases.
+#
+# H2 dumps should be created using the same (or compatible) version of
+# h2 that is used to run the db.
+#
+# Docspell 0.38.0 and earlier uses H2 1.4.x. From Docspell 0.39.0
+# onwards it's 2.1.x.
+#
+# Set the H2 version via an environment variable 'H2_VERSION'.
+# Additionally a user and password are required, set these via env
+# variables H2_USER and H2_PASSWORD. (or modify this script)
+#
+# Creating/restoring a dump requires to specify the database file. H2
+# appends suffixes like '.mv.db' and '.trace.db', but here the base
+# file is required. So if you see a file 'mydb.mv.db', specify here
+# 'mydb' only or a complete JDBC url.
+#
+# The target file or target db must not exist.
+#
+# The dump file contains (H2 specific) SQL that recreates the
+# database. It can be modified if necessary. This SQL script can then
+# be used to restore the database even to a newer version of H2.
+
+set -e
+
+h2_user=${H2_USER:-"sa"}
+h2_password=${H2_PASSWORD:-""}
+h2_version=${H2_VERSION:-"1.4.200"}
+#h2_version="2.1.214"
+
+h2_jar_url="https://search.maven.org/remotecontent?filepath=com/h2database/h2/$h2_version/h2-$h2_version.jar"
+h2_jdbc_opts=";MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"
+
+tempdir=$(mktemp -d "h2-util.XXXXX")
+trap "rm -rf $tempdir" EXIT
+
+
+prepare() {
+    echo "Prepare for h2 version: $h2_version"
+    curl -sSL -o $tempdir/h2.jar "$h2_jar_url"
+}
+
+create_dump() {
+    src_db_file="$1"
+    target_file="$2"
+
+    jdbc_url=""
+    if [[ "$src_db_file" =~ jdbc:h2:.* ]]; then
+        jdbc_url="$src_db_file"
+    elif [ -r "$src_db_file.mv.db" ]; then
+        jdbc_url="jdbc:h2://$(realpath "$src_db_file")$h2_jdbc_opts"
+    else
+        echo "Invalid database. Either specify the file or a full JDBC url."
+        echo "Usage: $0 dump <db-file|jdbc_url> <target-file>"
+        exit 1
+    fi
+
+    if [ -z "$target_file" ]; then
+        echo "No target file given"
+        echo "Usage: $0 dump <db-file|jdbc_url> <target-file>"
+        exit 1
+    fi
+    if [ -r "$target_file" ]; then
+        echo "The target file '$target_file' already exists!"
+        echo "Usage: $0 dump <db-file|jdbc_url> <target-file>"
+        exit 1
+    fi
+
+    echo "Creating a dump: $jdbc_url -> $target_file"
+    prepare
+    java -cp "$tempdir/h2.jar" org.h2.tools.Script \
+         -url "$jdbc_url" \
+         -user "$h2_user" \
+         -password "$h2_password" \
+         -script "$target_file"
+}
+
+restore_dump() {
+    backup_file="$1"
+    target_db_file="$2"
+
+    jdbc_url=""
+    if [[ "$target_db_file" =~ jdbc:h2:.* ]]; then
+        jdbc_url="$target_db_file"
+    elif ! [ -r "$target_db_file.mv.db" ]; then
+        jdbc_url="jdbc:h2://$(realpath "$target_db_file")$h2_jdbc_opts"
+    else
+        echo "Invalid database or it does already exist. Either specify the file or a full JDBC url."
+        echo "Usage: $0 restore <dump-file> <db-file|jdbc_url>"
+        exit 1
+    fi
+
+    if [ -z "$backup_file" ]; then
+        echo "No dump file given"
+        echo "Usage: $0 restore <dump-file> <db-file|jdbc_url>"
+        exit 1
+    fi
+    if ! [ -r "$backup_file" ]; then
+        echo "The dump file '$backup_file' doesn't exists!"
+        echo "Usage: $0 dump <db-file|jdbc_url> <target-file>"
+        exit 1
+    fi
+
+    echo "Restore a dump: $backup_file -> $jdbc_url"
+    prepare
+    java -cp "$tempdir/h2.jar" org.h2.tools.RunScript \
+         -url "$jdbc_url" \
+         -user "$h2_user" \
+         -password "$h2_password" \
+         -script "$backup_file" \
+         -options FROM_1X
+}
+
+case "$1" in
+    dump)
+        shift
+        create_dump "$@"
+        ;;
+
+    restore)
+        echo "Restoring from a file"
+        shift
+        restore_dump "$@"
+        ;;
+
+    *)
+        echo "Invalid command. One of: dump, restore"
+        exit 1
+esac