Use different file stores based on config

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -6,9 +6,13 @@
 
 package docspell.backend
 
+import cats.data.{Validated, ValidatedNec}
+import cats.implicits._
+
 import docspell.backend.signup.{Config => SignupConfig}
 import docspell.common._
 import docspell.store.JdbcConfig
+import docspell.store.file.FileRepositoryConfig
 
 import emil.javamail.Settings
 
@@ -21,10 +25,49 @@ case class Config(
 
   def mailSettings: Settings =
     Settings.defaultSettings.copy(debug = mailDebug)
+
 }
 
 object Config {
 
-  case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType])
+  case class Files(
+      chunkSize: Int,
+      validMimeTypes: Seq[MimeType],
+      defaultStore: Ident,
+      stores: Map[Ident, FileStoreConfig]
+  ) {
+    val enabledStores: Map[Ident, FileStoreConfig] =
+      stores.view.filter(_._2.enabled).toMap
 
+    def defaultStoreConfig: FileStoreConfig =
+      enabledStores(defaultStore)
+
+    def toFileRepositoryConfig: FileRepositoryConfig =
+      defaultStoreConfig match {
+        case FileStoreConfig.DefaultDatabase(_) =>
+          FileRepositoryConfig.Database(chunkSize)
+        case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) =>
+          FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize)
+        case FileStoreConfig.FileSystem(_, directory) =>
+          FileRepositoryConfig.Directory(directory, chunkSize)
+      }
+
+    def validate: ValidatedNec[String, Files] = {
+      val storesEmpty =
+        if (enabledStores.isEmpty)
+          Validated.invalidNec(
+            "No file stores defined! Make sure at least one enabled store is present."
+          )
+        else Validated.validNec(())
+
+      val defaultStorePresent =
+        enabledStores.get(defaultStore) match {
+          case Some(_) => Validated.validNec(())
+          case None =>
+            Validated.invalidNec(s"Default file store not present: ${defaultStore.id}")
+        }
+
+      (storesEmpty |+| defaultStorePresent).map(_ => this)
+    }
+  }
 }

modules/common/src/main/scala/docspell/common/FileStoreConfig.scala

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import fs2.io.file.Path
+
+sealed trait FileStoreConfig {
+  def enabled: Boolean
+  def storeType: FileStoreType
+}
+object FileStoreConfig {
+  case class DefaultDatabase(enabled: Boolean) extends FileStoreConfig {
+    val storeType = FileStoreType.DefaultDatabase
+  }
+
+  case class FileSystem(
+      enabled: Boolean,
+      directory: Path
+  ) extends FileStoreConfig {
+    val storeType = FileStoreType.FileSystem
+  }
+
+  case class S3(
+      enabled: Boolean,
+      endpoint: String,
+      accessKey: String,
+      secretKey: String,
+      bucket: String
+  ) extends FileStoreConfig {
+    val storeType = FileStoreType.S3
+  }
+}

modules/common/src/main/scala/docspell/common/FileStoreType.scala

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import cats.data.NonEmptyList
+
+sealed trait FileStoreType { self: Product =>
+  def name: String =
+    productPrefix.toLowerCase
+}
+object FileStoreType {
+  case object DefaultDatabase extends FileStoreType
+
+  case object S3 extends FileStoreType
+
+  case object FileSystem extends FileStoreType
+
+  val all: NonEmptyList[FileStoreType] =
+    NonEmptyList.of(DefaultDatabase, S3, FileSystem)
+
+  def fromString(str: String): Either[String, FileStoreType] =
+    all
+      .find(_.name.equalsIgnoreCase(str))
+      .toRight(s"Invalid file store type: $str")
+
+  def unsafeFromString(str: String): FileStoreType =
+    fromString(str).fold(sys.error, identity)
+}

modules/config/src/main/scala/docspell/config/Implicits.scala

@@ -18,9 +18,18 @@ import docspell.logging.{Level, LogConfig}
 import com.github.eikek.calev.CalEvent
 import pureconfig.ConfigReader
 import pureconfig.error.{CannotConvert, FailureReason}
+import pureconfig.generic.{CoproductHint, FieldCoproductHint}
 import scodec.bits.ByteVector
 
 object Implicits {
+  // the value "s-3" looks strange, this is to allow to write "s3" in the config
+  implicit val fileStoreCoproductHint: CoproductHint[FileStoreConfig] =
+    new FieldCoproductHint[FileStoreConfig]("type") {
+      override def fieldValue(name: String) =
+        if (name.equalsIgnoreCase("S3")) "s3"
+        else super.fieldValue(name)
+    }
+
   implicit val accountIdReader: ConfigReader[AccountId] =
     ConfigReader[String].emap(reason(AccountId.parse))
 
@@ -42,6 +51,9 @@ object Implicits {
   implicit val identReader: ConfigReader[Ident] =
     ConfigReader[String].emap(reason(Ident.fromString))
 
+  implicit def identMapReader[B: ConfigReader]: ConfigReader[Map[Ident, B]] =
+    pureconfig.configurable.genericMapReader[Ident, B](reason(Ident.fromString))
+
   implicit val byteVectorReader: ConfigReader[ByteVector] =
     ConfigReader[String].emap(reason { str =>
       if (str.startsWith("hex:"))
@@ -70,6 +82,9 @@ object Implicits {
   implicit val logLevelReader: ConfigReader[Level] =
     ConfigReader[String].emap(reason(Level.fromString))
 
+  implicit val fileStoreTypeReader: ConfigReader[FileStoreType] =
+    ConfigReader[String].emap(reason(FileStoreType.fromString))
+
   def reason[A: ClassTag](
       f: String => Either[String, A]
   ): String => Either[FailureReason, A] =

modules/joex/src/main/resources/reference.conf

@@ -646,6 +646,41 @@ Docpell Update Check
     # restrict file types that should be handed over to processing.
     # By default all files are allowed.
     valid-mime-types = [ ]
+
+    # The id of an enabled store from the `stores` array that should
+    # be used.
+    #
+    # IMPORTANT NOTE: All nodes must have the exact same file store
+    # configuration!
+    default-store = "database"
+
+    # A list of possible file stores. Each entry must have a unique
+    # id. The `type` is one of: default-database, filesystem, s3.
+    #
+    # The enabled property serves currently to define target stores
+    # for te "copy files" task. All stores with enabled=false are
+    # removed from the list. The `default-store` must be enabled.
+    stores = {
+      database =
+        { enabled = true
+          type = "default-database"
+        }
+
+      filesystem =
+        { enabled = false
+          type = "file-system"
+          directory = "/some/directory"
+        }
+
+      minio =
+       { enabled = false
+         type = "s3"
+         endpoint = "http://localhost:9000"
+         access-key = "username"
+         secret-key = "password"
+         bucket = "docspell"
+       }
+    }
   }
 
   # Configuration of the full-text search engine.

modules/joex/src/main/scala/docspell/joex/ConfigFile.scala

@@ -19,8 +19,6 @@ import pureconfig.generic.auto._
 import yamusca.imports._
 
 object ConfigFile {
-  import Implicits._
-
   def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
     val logger = docspell.logging.getLogger[F]
     ConfigFactory
@@ -51,6 +49,7 @@ object ConfigFile {
       Validation.failWhen(
         cfg => cfg.updateCheck.enabled && cfg.updateCheck.subject.els.isEmpty,
         "No subject given for enabled update check!"
-      )
+      ),
+      Validation(cfg => cfg.files.validate.map(_ => cfg))
     )
 }

modules/joex/src/main/scala/docspell/joex/JoexServer.scala

@@ -16,7 +16,6 @@ import docspell.common.Pools
 import docspell.joex.routes._
 import docspell.pubsub.naive.NaivePubSub
 import docspell.store.Store
-import docspell.store.file.FileRepositoryConfig
 import docspell.store.records.RInternalSetting
 
 import org.http4s.HttpApp
@@ -42,7 +41,7 @@ object JoexServer {
 
       store <- Store.create[F](
         cfg.jdbc,
-        FileRepositoryConfig.Database(cfg.files.chunkSize),
+        cfg.files.toFileRepositoryConfig,
         pools.connectEC
       )
       settings <- Resource.eval(store.transact(RInternalSetting.create))

modules/restserver/src/main/resources/reference.conf

@@ -358,6 +358,41 @@ docspell.server {
       # restrict file types that should be handed over to processing.
       # By default all files are allowed.
       valid-mime-types = [ ]
+
+      # The id of an enabled store from the `stores` array that should
+      # be used.
+      #
+      # IMPORTANT NOTE: All nodes must have the exact same file store
+      # configuration!
+      default-store = "database"
+
+      # A list of possible file stores. Each entry must have a unique
+      # id. The `type` is one of: default-database, filesystem, s3.
+      #
+      # The enabled property serves currently to define target stores
+      # for te "copy files" task. All stores with enabled=false are
+      # removed from the list. The `default-store` must be enabled.
+      stores = {
+        database =
+          { enabled = true
+            type = "default-database"
+          }
+
+        filesystem =
+          { enabled = false
+            type = "file-system"
+            directory = "/some/directory"
+          }
+
+        minio =
+         { enabled = false
+           type = "s3"
+           endpoint = "http://localhost:9000"
+           access-key = "username"
+           secret-key = "password"
+           bucket = "docspell"
+         }
+      }
     }
   }
 }

modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala

@@ -24,12 +24,18 @@ import scodec.bits.ByteVector
 object ConfigFile {
   private[this] val unsafeLogger = docspell.logging.unsafeLogger
 
+  // IntelliJ is wrong, this is required
   import Implicits._
 
   def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
     val logger = docspell.logging.getLogger[F]
     val validate =
-      Validation.of(generateSecretIfEmpty, duplicateOpenIdProvider, signKeyVsUserUrl)
+      Validation.of(
+        generateSecretIfEmpty,
+        duplicateOpenIdProvider,
+        signKeyVsUserUrl,
+        filesValidate
+      )
     ConfigFactory
       .default[F, Config](logger, "docspell.server")(args, validate)
   }
@@ -97,4 +103,7 @@ object ConfigFile {
             .map(checkProvider)
         )
     }
+
+  def filesValidate: Validation[Config] =
+    Validation(cfg => cfg.backend.files.validate.map(_ => cfg))
 }

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -18,7 +18,6 @@ import docspell.restserver.http4s.InternalHeader
 import docspell.restserver.ws.OutputEvent.KeepAlive
 import docspell.restserver.ws.OutputEvent
 import docspell.store.Store
-import docspell.store.file.FileRepositoryConfig
 import docspell.store.records.RInternalSetting
 import org.http4s._
 import org.http4s.blaze.client.BlazeClientBuilder
@@ -74,7 +73,7 @@ object RestServer {
       httpClient <- BlazeClientBuilder[F].resource
       store <- Store.create[F](
         cfg.backend.jdbc,
-        FileRepositoryConfig.Database(cfg.backend.files.chunkSize),
+        cfg.backend.files.toFileRepositoryConfig,
         pools.connectEC
       )
       setting <- Resource.eval(store.transact(RInternalSetting.create))