From e82b00c582a98d6509fbd454c013e6253fa0adff Mon Sep 17 00:00:00 2001 From: eikek Date: Mon, 7 Mar 2022 17:21:38 +0100 Subject: [PATCH] Use different file stores based on config --- .../main/scala/docspell/backend/Config.scala | 45 ++++++++++++++++++- .../docspell/common/FileStoreConfig.scala | 36 +++++++++++++++ .../scala/docspell/common/FileStoreType.scala | 32 +++++++++++++ .../scala/docspell/config/Implicits.scala | 15 +++++++ .../joex/src/main/resources/reference.conf | 35 +++++++++++++++ .../main/scala/docspell/joex/ConfigFile.scala | 5 +-- .../main/scala/docspell/joex/JoexServer.scala | 3 +- .../src/main/resources/reference.conf | 35 +++++++++++++++ .../docspell/restserver/ConfigFile.scala | 11 ++++- .../docspell/restserver/RestServer.scala | 3 +- 10 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 modules/common/src/main/scala/docspell/common/FileStoreConfig.scala create mode 100644 modules/common/src/main/scala/docspell/common/FileStoreType.scala diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index 8ddce838..e9cd3356 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -6,9 +6,13 @@ package docspell.backend +import cats.data.{Validated, ValidatedNec} +import cats.implicits._ + import docspell.backend.signup.{Config => SignupConfig} import docspell.common._ import docspell.store.JdbcConfig +import docspell.store.file.FileRepositoryConfig import emil.javamail.Settings @@ -21,10 +25,49 @@ case class Config( def mailSettings: Settings = Settings.defaultSettings.copy(debug = mailDebug) + } object Config { - case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) + case class Files( + chunkSize: Int, + validMimeTypes: Seq[MimeType], + defaultStore: Ident, + stores: Map[Ident, FileStoreConfig] + ) { + val enabledStores: Map[Ident, FileStoreConfig] = + stores.view.filter(_._2.enabled).toMap + def defaultStoreConfig: FileStoreConfig = + enabledStores(defaultStore) + + def toFileRepositoryConfig: FileRepositoryConfig = + defaultStoreConfig match { + case FileStoreConfig.DefaultDatabase(_) => + FileRepositoryConfig.Database(chunkSize) + case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) => + FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) + case FileStoreConfig.FileSystem(_, directory) => + FileRepositoryConfig.Directory(directory, chunkSize) + } + + def validate: ValidatedNec[String, Files] = { + val storesEmpty = + if (enabledStores.isEmpty) + Validated.invalidNec( + "No file stores defined! Make sure at least one enabled store is present." + ) + else Validated.validNec(()) + + val defaultStorePresent = + enabledStores.get(defaultStore) match { + case Some(_) => Validated.validNec(()) + case None => + Validated.invalidNec(s"Default file store not present: ${defaultStore.id}") + } + + (storesEmpty |+| defaultStorePresent).map(_ => this) + } + } } diff --git a/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala new file mode 100644 index 00000000..80652217 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import fs2.io.file.Path + +sealed trait FileStoreConfig { + def enabled: Boolean + def storeType: FileStoreType +} +object FileStoreConfig { + case class DefaultDatabase(enabled: Boolean) extends FileStoreConfig { + val storeType = FileStoreType.DefaultDatabase + } + + case class FileSystem( + enabled: Boolean, + directory: Path + ) extends FileStoreConfig { + val storeType = FileStoreType.FileSystem + } + + case class S3( + enabled: Boolean, + endpoint: String, + accessKey: String, + secretKey: String, + bucket: String + ) extends FileStoreConfig { + val storeType = FileStoreType.S3 + } +} diff --git a/modules/common/src/main/scala/docspell/common/FileStoreType.scala b/modules/common/src/main/scala/docspell/common/FileStoreType.scala new file mode 100644 index 00000000..f67b218d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileStoreType.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList + +sealed trait FileStoreType { self: Product => + def name: String = + productPrefix.toLowerCase +} +object FileStoreType { + case object DefaultDatabase extends FileStoreType + + case object S3 extends FileStoreType + + case object FileSystem extends FileStoreType + + val all: NonEmptyList[FileStoreType] = + NonEmptyList.of(DefaultDatabase, S3, FileSystem) + + def fromString(str: String): Either[String, FileStoreType] = + all + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid file store type: $str") + + def unsafeFromString(str: String): FileStoreType = + fromString(str).fold(sys.error, identity) +} diff --git a/modules/config/src/main/scala/docspell/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala index 0bc7dda4..e9b23348 100644 --- a/modules/config/src/main/scala/docspell/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -18,9 +18,18 @@ import docspell.logging.{Level, LogConfig} import com.github.eikek.calev.CalEvent import pureconfig.ConfigReader import pureconfig.error.{CannotConvert, FailureReason} +import pureconfig.generic.{CoproductHint, FieldCoproductHint} import scodec.bits.ByteVector object Implicits { + // the value "s-3" looks strange, this is to allow to write "s3" in the config + implicit val fileStoreCoproductHint: CoproductHint[FileStoreConfig] = + new FieldCoproductHint[FileStoreConfig]("type") { + override def fieldValue(name: String) = + if (name.equalsIgnoreCase("S3")) "s3" + else super.fieldValue(name) + } + implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) @@ -42,6 +51,9 @@ object Implicits { implicit val identReader: ConfigReader[Ident] = ConfigReader[String].emap(reason(Ident.fromString)) + implicit def identMapReader[B: ConfigReader]: ConfigReader[Map[Ident, B]] = + pureconfig.configurable.genericMapReader[Ident, B](reason(Ident.fromString)) + implicit val byteVectorReader: ConfigReader[ByteVector] = ConfigReader[String].emap(reason { str => if (str.startsWith("hex:")) @@ -70,6 +82,9 @@ object Implicits { implicit val logLevelReader: ConfigReader[Level] = ConfigReader[String].emap(reason(Level.fromString)) + implicit val fileStoreTypeReader: ConfigReader[FileStoreType] = + ConfigReader[String].emap(reason(FileStoreType.fromString)) + def reason[A: ClassTag]( f: String => Either[String, A] ): String => Either[FailureReason, A] = diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 8516c916..444ef5ef 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -646,6 +646,41 @@ Docpell Update Check # restrict file types that should be handed over to processing. # By default all files are allowed. valid-mime-types = [ ] + + # The id of an enabled store from the `stores` array that should + # be used. + # + # IMPORTANT NOTE: All nodes must have the exact same file store + # configuration! + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # The enabled property serves currently to define target stores + # for te "copy files" task. All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "docspell" + } + } } # Configuration of the full-text search engine. diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index a3663030..32049b16 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -19,8 +19,6 @@ import pureconfig.generic.auto._ import yamusca.imports._ object ConfigFile { - import Implicits._ - def loadConfig[F[_]: Async](args: List[String]): F[Config] = { val logger = docspell.logging.getLogger[F] ConfigFactory @@ -51,6 +49,7 @@ object ConfigFile { Validation.failWhen( cfg => cfg.updateCheck.enabled && cfg.updateCheck.subject.els.isEmpty, "No subject given for enabled update check!" - ) + ), + Validation(cfg => cfg.files.validate.map(_ => cfg)) ) } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 1d0458d4..29b7b9e8 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -16,7 +16,6 @@ import docspell.common.Pools import docspell.joex.routes._ import docspell.pubsub.naive.NaivePubSub import docspell.store.Store -import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s.HttpApp @@ -42,7 +41,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, - FileRepositoryConfig.Database(cfg.files.chunkSize), + cfg.files.toFileRepositoryConfig, pools.connectEC ) settings <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 14b21372..d3306679 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -358,6 +358,41 @@ docspell.server { # restrict file types that should be handed over to processing. # By default all files are allowed. valid-mime-types = [ ] + + # The id of an enabled store from the `stores` array that should + # be used. + # + # IMPORTANT NOTE: All nodes must have the exact same file store + # configuration! + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # The enabled property serves currently to define target stores + # for te "copy files" task. All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "docspell" + } + } } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala index 5e72a1d4..2e225cae 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala @@ -24,12 +24,18 @@ import scodec.bits.ByteVector object ConfigFile { private[this] val unsafeLogger = docspell.logging.unsafeLogger + // IntelliJ is wrong, this is required import Implicits._ def loadConfig[F[_]: Async](args: List[String]): F[Config] = { val logger = docspell.logging.getLogger[F] val validate = - Validation.of(generateSecretIfEmpty, duplicateOpenIdProvider, signKeyVsUserUrl) + Validation.of( + generateSecretIfEmpty, + duplicateOpenIdProvider, + signKeyVsUserUrl, + filesValidate + ) ConfigFactory .default[F, Config](logger, "docspell.server")(args, validate) } @@ -97,4 +103,7 @@ object ConfigFile { .map(checkProvider) ) } + + def filesValidate: Validation[Config] = + Validation(cfg => cfg.backend.files.validate.map(_ => cfg)) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 57c77707..4546fc34 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -18,7 +18,6 @@ import docspell.restserver.http4s.InternalHeader import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.restserver.ws.OutputEvent import docspell.store.Store -import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder @@ -74,7 +73,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, - FileRepositoryConfig.Database(cfg.backend.files.chunkSize), + cfg.backend.files.toFileRepositoryConfig, pools.connectEC ) setting <- Resource.eval(store.transact(RInternalSetting.create))