Merge pull request #247 from eikek/fix-db-migration

Fix db migration

modules/common/src/main/scala/docspell/common/ItemState.scala

@@ -23,6 +23,11 @@ object ItemState {
   case object Created    extends ItemState
   case object Confirmed  extends ItemState
 
+  def premature: ItemState  = Premature
+  def processing: ItemState = Processing
+  def created: ItemState    = Created
+  def confirmed: ItemState  = Confirmed
+
   def fromString(str: String): Either[String, ItemState] =
     str.toLowerCase match {
       case "premature"  => Right(Premature)

modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala

@@ -157,8 +157,9 @@ object RegexNerFile {
           .reduce(_ ++ fr"UNION ALL" ++ _)
 
       selectSimple(fr"MAX(t)", fr"(" ++ sql ++ fr") as x", Fragment.empty)
-        .query[Timestamp]
+        .query[Option[Timestamp]]
         .option
+        .map(_.flatten)
     }
   }
 }

modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala

@@ -98,11 +98,14 @@ object TextAnalysis {
       ctx: Context[F, Args],
       cfg: Config.TextAnalysis
   ): OptionT[F, Ident] =
-    if (cfg.classification.enabled)
-      OptionT(ctx.store.transact(RClassifierSetting.findById(ctx.args.meta.collective)))
-        .filter(_.enabled)
-        .mapFilter(_.fileId)
-    else
-      OptionT.none
+    (if (cfg.classification.enabled)
+       OptionT(ctx.store.transact(RClassifierSetting.findById(ctx.args.meta.collective)))
+         .filter(_.enabled)
+         .mapFilter(_.fileId)
+     else
+       OptionT.none[F, Ident]).orElse(
+      OptionT.liftF(ctx.logger.info("Classification is disabled.")) *> OptionT
+        .none[F, Ident]
+    )
 
 }

modules/store/src/main/resources/db/migration/h2/V1.0.0__initial_database.sql

@@ -0,0 +1,210 @@
+CREATE TABLE "filemeta" (
+  "id" varchar(254) not null primary key,
+  "timestamp" varchar(40) not null,
+  "mimetype" varchar(254) not null,
+  "length" bigint not null,
+  "checksum" varchar(254) not null,
+  "chunks" int not null,
+  "chunksize" int not null
+);
+
+CREATE TABLE "filechunk" (
+  fileId varchar(254) not null,
+  chunkNr int not null,
+  chunkLength int not null,
+  chunkData bytea not null,
+  primary key (fileId, chunkNr)
+);
+
+CREATE TABLE "collective" (
+  "cid" varchar(254) not null primary key,
+  "state" varchar(254) not null,
+  "doclang" varchar(254) not null,
+  "created" timestamp not null
+);
+
+CREATE TABLE "user_" (
+  "uid" varchar(254) not null primary key,
+  "login" varchar(254) not null,
+  "cid" varchar(254) not null,
+  "password" varchar(254) not null,
+  "state" varchar(254) not null,
+  "email" varchar(254),
+  "logincount" int not null,
+  "lastlogin" timestamp,
+  "created" timestamp not null,
+  unique ("cid", "login"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "invitation" (
+  "id" varchar(254) not null primary key,
+  "created" timestamp not null
+);
+
+CREATE TABLE "source" (
+  "sid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "abbrev" varchar(254) not null,
+  "description" text,
+  "counter" int not null,
+  "enabled" boolean not null,
+  "priority" int not null,
+  "created" timestamp not null,
+  unique ("cid", "abbrev"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "organization" (
+  "oid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "street" varchar(254),
+  "zip" varchar(254),
+  "city" varchar(254),
+  "country" varchar(254),
+  "notes" text,
+  "created" timestamp not null,
+  unique ("cid", "name"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "person" (
+  "pid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "street" varchar(254),
+  "zip" varchar(254),
+  "city" varchar(254),
+  "country" varchar(254),
+  "notes" text,
+  "concerning" boolean not null,
+  "created" varchar(30) not null,
+  unique ("cid", "name"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "contact" (
+  "contactid" varchar(254) not null primary key,
+  "value" varchar(254) not null,
+  "kind" varchar(254) not null,
+  "pid" varchar(254),
+  "oid" varchar(254),
+  "created" timestamp not null,
+  foreign key ("pid") references "person"("pid"),
+  foreign key ("oid") references "organization"("oid")
+);
+
+CREATE TABLE "equipment" (
+  "eid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "created" timestamp not null,
+  unique ("cid","eid"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "item" (
+  "itemid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "itemdate" timestamp,
+  "source" varchar(254) not null,
+  "incoming" boolean not null,
+  "state" varchar(254) not null,
+  "corrorg" varchar(254),
+  "corrperson" varchar(254),
+  "concperson" varchar(254),
+  "concequipment" varchar(254),
+  "inreplyto" varchar(254),
+  "duedate" timestamp,
+  "notes" text,
+  "created" timestamp not null,
+  "updated" timestamp not null,
+  foreign key ("inreplyto") references "item"("itemid"),
+  foreign key ("corrorg") references "organization"("oid"),
+  foreign key ("corrperson") references "person"("pid"),
+  foreign key ("concperson") references "person"("pid"),
+  foreign key ("concequipment") references "equipment"("eid"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "attachment" (
+  "attachid" varchar(254) not null primary key,
+  "itemid" varchar(254) not null,
+  "filemetaid" varchar(254) not null,
+  "position" int not null,
+  "created" timestamp not null,
+  "name" varchar(254),
+  foreign key ("itemid") references "item"("itemid"),
+  foreign key ("filemetaid") references "filemeta"("id")
+);
+
+CREATE TABLE "attachmentmeta" (
+  "attachid" varchar(254) not null primary key,
+  "content" text,
+  "nerlabels" text,
+  "itemproposals" text,
+  foreign key ("attachid") references "attachment"("attachid")
+);
+
+CREATE TABLE "tag" (
+  "tid" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "category" varchar(254),
+  "created" timestamp not null,
+  unique ("cid", "name"),
+  foreign key ("cid") references "collective"("cid")
+);
+
+CREATE TABLE "tagitem" (
+  "tagitemid" varchar(254) not null primary key,
+  "itemid" varchar(254) not null,
+  "tid" varchar(254) not null,
+  unique ("itemid", "tid"),
+  foreign key ("itemid") references "item"("itemid"),
+  foreign key ("tid") references "tag"("tid")
+);
+
+CREATE TABLE "job" (
+  "jid" varchar(254) not null primary key,
+  "task" varchar(254) not null,
+  "group_" varchar(254) not null,
+  "args" text not null,
+  "subject" varchar(254) not null,
+  "submitted" timestamp not null,
+  "submitter" varchar(254) not null,
+  "priority" int not null,
+  "state" varchar(254) not null,
+  "retries" int not null,
+  "progress" int not null,
+  "tracker" varchar(254),
+  "worker" varchar(254),
+  "started" timestamp,
+  "finished" timestamp,
+  "startedmillis" bigint
+);
+
+CREATE TABLE "joblog" (
+  "id" varchar(254) not null primary key,
+  "jid" varchar(254) not null,
+  "level" varchar(254) not null,
+  "created" timestamp not null,
+  "message" text not null,
+  foreign key ("jid") references "job"("jid")
+);
+
+CREATE TABLE "jobgroupuse" (
+  "groupid" varchar(254) not null,
+  "workerid" varchar(254) not null,
+  primary key ("groupid", "workerid")
+);
+
+CREATE TABLE "node" (
+  "id" varchar(254) not null,
+  "type" varchar(254) not null,
+  "url" varchar(254) not null,
+  "updated" timestamp not null,
+  "created" timestamp not null
+)

modules/store/src/main/resources/db/migration/h2/V1.1.0__useremail.sql

@@ -0,0 +1,39 @@
+CREATE TABLE "useremail" (
+  "id" varchar(254) not null primary key,
+  "uid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "smtp_host" varchar(254) not null,
+  "smtp_port" int,
+  "smtp_user" varchar(254),
+  "smtp_password" varchar(254),
+  "smtp_ssl" varchar(254) not null,
+  "smtp_certcheck" boolean not null,
+  "mail_from" varchar(254) not null,
+  "mail_replyto" varchar(254),
+  "created" timestamp not null,
+  unique ("uid", "name"),
+  foreign key ("uid") references "user_"("uid")
+);
+
+CREATE TABLE "sentmail" (
+  "id" varchar(254) not null primary key,
+  "uid" varchar(254) not null,
+  "message_id" varchar(254) not null,
+  "sender" varchar(254) not null,
+  "conn_name" varchar(254) not null,
+  "subject" varchar(254) not null,
+  "recipients" varchar(254) not null,
+  "body" text not null,
+  "created" timestamp not null,
+  foreign key("uid") references "user_"("uid")
+);
+
+CREATE TABLE "sentmailitem" (
+  "id" varchar(254) not null primary key,
+  "item_id" varchar(254) not null,
+  "sentmail_id" varchar(254) not null,
+  "created" timestamp not null,
+  unique ("item_id", "sentmail_id"),
+  foreign key("item_id") references "item"("itemid"),
+  foreign key("sentmail_id") references "sentmail"("id")
+);

modules/store/src/main/resources/db/migration/h2/V1.2.0__origin_source.sql

@@ -0,0 +1,11 @@
+CREATE TABLE "attachment_source" (
+  "id" varchar(254) not null primary key,
+  "file_id" varchar(254) not null,
+  "filename" varchar(254),
+  "created" timestamp not null,
+  foreign key ("file_id") references "filemeta"("id"),
+  foreign key ("id") references "attachment"("attachid")
+);
+
+INSERT INTO "attachment_source"
+  SELECT "attachid","filemetaid","name","created" FROM "attachment";

modules/store/src/main/resources/db/migration/h2/V1.3.0__periodic_job.sql

@@ -0,0 +1,18 @@
+CREATE TABLE "periodic_task" (
+  "id" varchar(254) not null primary key,
+  "enabled" boolean not null,
+  "task" varchar(254) not null,
+  "group_" varchar(254) not null,
+  "args" text not null,
+  "subject" varchar(254) not null,
+  "submitter" varchar(254) not null,
+  "priority" int not null,
+  "worker" varchar(254),
+  "marked" timestamp,
+  "timer" varchar(254) not null,
+  "nextrun" timestamp not null,
+  "created" timestamp not null
+);
+
+CREATE INDEX "periodic_task_nextrun_idx" ON "periodic_task"("nextrun");
+CREATE INDEX "periodic_task_worker_idx" ON "periodic_task"("worker");

modules/store/src/main/resources/db/migration/h2/V1.4.0__attachment_archive.sql

@@ -0,0 +1,12 @@
+CREATE TABLE "attachment_archive" (
+  "id" varchar(254) not null primary key,
+  "file_id" varchar(254) not null,
+  "filename" varchar(254),
+  "message_id" varchar(254),
+  "created" timestamp not null,
+  foreign key ("file_id") references "filemeta"("id"),
+  foreign key ("id") references "attachment"("attachid")
+);
+
+CREATE INDEX "attachment_archive_message_id_idx"
+ON "attachment_archive"("message_id");

modules/store/src/main/resources/db/migration/h2/V1.5.0__userimap.sql

@@ -0,0 +1,14 @@
+CREATE TABLE "userimap" (
+  "id" varchar(254) not null primary key,
+  "uid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "imap_host" varchar(254) not null,
+  "imap_port" int,
+  "imap_user" varchar(254),
+  "imap_password" varchar(254),
+  "imap_ssl" varchar(254) not null,
+  "imap_certcheck" boolean not null,
+  "created" timestamp not null,
+  unique ("uid", "name"),
+  foreign key ("uid") references "user_"("uid")
+);

modules/store/src/main/resources/db/migration/h2/V1.6.0__integration_enabled.sql

@@ -0,0 +1,7 @@
+ALTER TABLE "collective"
+ADD COLUMN "integration_enabled" BOOLEAN;
+
+UPDATE "collective" SET "integration_enabled" = true;
+
+ALTER TABLE "collective"
+ALTER COLUMN "integration_enabled" SET NOT NULL;

modules/store/src/main/resources/db/migration/h2/V1.7.0__fts-migration.sql

@@ -0,0 +1,10 @@
+CREATE TABLE "fts_migration" (
+  "id" varchar(254) not null primary key,
+  "version" int not null,
+  "fts_engine" varchar(254) not null,
+  "description" varchar(254) not null,
+  "created" timestamp not null
+);
+
+CREATE UNIQUE INDEX "fts_migration_version_engine_idx"
+ON "fts_migration"("version", "fts_engine");

modules/store/src/main/resources/db/migration/h2/V1.8.0__folders.sql

@@ -0,0 +1,34 @@
+CREATE TABLE "folder" (
+  "id" varchar(254) not null primary key,
+  "name" varchar(254) not null,
+  "cid" varchar(254) not null,
+  "owner" varchar(254) not null,
+  "created" timestamp not null,
+  unique ("name", "cid"),
+  foreign key ("cid") references "collective"("cid"),
+  foreign key ("owner") references "user_"("uid")
+);
+
+CREATE TABLE "folder_member" (
+  "id" varchar(254) not null primary key,
+  "folder_id" varchar(254) not null,
+  "user_id" varchar(254) not null,
+  "created" timestamp not null,
+  unique ("folder_id", "user_id"),
+  foreign key ("folder_id") references "folder"("id"),
+  foreign key ("user_id") references "user_"("uid")
+);
+
+ALTER TABLE "item"
+ADD COLUMN "folder_id" varchar(254) NULL;
+
+ALTER TABLE "item"
+ADD FOREIGN KEY ("folder_id")
+REFERENCES "folder"("id");
+
+ALTER TABLE "source"
+ADD COLUMN "folder_id" varchar(254) NULL;
+
+ALTER TABLE "source"
+ADD FOREIGN KEY ("folder_id")
+REFERENCES "folder"("id");

modules/store/src/main/resources/db/migration/h2/V1.9.0__updated_column.sql

@@ -0,0 +1,29 @@
+-- organization
+ALTER TABLE "organization"
+ADD COLUMN "updated" timestamp;
+
+UPDATE "organization" SET "updated" = "created";
+
+ALTER TABLE "organization"
+ALTER COLUMN "updated" SET NOT NULL;
+
+-- person
+ALTER TABLE "person" ALTER COLUMN "created"
+  TYPE timestamp;
+
+ALTER TABLE "person"
+ADD COLUMN "updated" timestamp;
+
+UPDATE "person" SET "updated" = "created";
+
+ALTER TABLE "person"
+ALTER COLUMN "updated" SET NOT NULL;
+
+-- equipment
+ALTER TABLE "equipment"
+ADD COLUMN "updated" timestamp;
+
+UPDATE "equipment" SET "updated" = "created";
+
+ALTER TABLE "equipment"
+ALTER COLUMN "updated" SET NOT NULL;

modules/store/src/main/resources/db/migration/h2/V1.9.1__classifier.sql

@@ -0,0 +1,11 @@
+CREATE TABLE "classifier_setting" (
+  "cid" varchar(254) not null primary key,
+  "enabled" boolean not null,
+  "schedule" varchar(254) not null,
+  "category" varchar(254) not null,
+  "item_count" int not null,
+  "file_id" varchar(254),
+  "created" timestamp not null,
+  foreign key ("cid") references "collective"("cid"),
+  foreign key ("file_id") references "filemeta"("id")
+);

modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala

@@ -15,13 +15,12 @@ object FlywayMigrate {
       logger.info("Running db migrations...")
       val locations = jdbc.dbmsName match {
         case Some(dbtype) =>
-          val name = if (dbtype == "h2") "postgresql" else dbtype
-          List(s"classpath:db/migration/${name}")
+          List(s"classpath:db/migration/${dbtype}")
         case None =>
           logger.warn(
-            s"Cannot read database name from jdbc url: ${jdbc.url}. Go with PostgreSQL"
+            s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2"
           )
-          List("classpath:db/postgresql")
+          List("classpath:db/h2")
       }
 
       logger.info(s"Using migration locations: $locations")

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -621,8 +621,14 @@ object QItem {
       collective: Ident,
       chunkSize: Int
   ): Stream[ConnectionIO, Ident] = {
-    val cols = Seq(RItem.Columns.id)
-    (selectSimple(cols, RItem.table, RItem.Columns.cid.is(collective)) ++
+    val cols   = Seq(RItem.Columns.id)
+    val iColl  = RItem.Columns.cid
+    val iState = RItem.Columns.state
+    (selectSimple(
+      cols,
+      RItem.table,
+      and(iColl.is(collective), iState.is(ItemState.confirmed))
+    ) ++
       orderBy(RItem.Columns.created.desc))
       .query[Ident]
       .streamWithChunkSize(chunkSize)