diff --git a/.scalafmt.conf b/.scalafmt.conf index fe959dae..28250946 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -2,6 +2,7 @@ version = "3.0.6" preset = default align.preset = some +runner.dialect = scala213 maxColumn = 90 diff --git a/build.sbt b/build.sbt index 9e1b0777..2d22c7b3 100644 --- a/build.sbt +++ b/build.sbt @@ -293,10 +293,23 @@ val common = project Dependencies.circe ++ Dependencies.loggingApi ++ Dependencies.calevCore ++ - Dependencies.calevCirce ++ - Dependencies.pureconfig.map(_ % "optional") + Dependencies.calevCirce ) +val config = project + .in(file("modules/config")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettingsMUnit) + .settings( + name := "docspell-config", + addCompilerPlugin(Dependencies.kindProjectorPlugin), + libraryDependencies ++= + Dependencies.fs2 ++ + Dependencies.pureconfig + ) + .dependsOn(common) + // Some example files for testing // https://file-examples.com/index.php/sample-documents-download/sample-doc-download/ val files = project @@ -603,7 +616,17 @@ val joex = project ), Revolver.enableDebugging(port = 5051, suspend = false) ) - .dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) + .dependsOn( + config, + store, + backend, + extract, + convert, + analysis, + joexapi, + restapi, + ftssolr + ) val restserver = project .in(file("modules/restserver")) @@ -666,7 +689,7 @@ val restserver = project } } ) - .dependsOn(restapi, joexapi, backend, webapp, ftssolr, oidc) + .dependsOn(config, restapi, joexapi, backend, webapp, ftssolr, oidc) // --- Website Documentation @@ -731,6 +754,7 @@ val root = project ) .aggregate( common, + config, extract, convert, analysis, diff --git a/modules/config/src/main/scala/docspell/config/ConfigFactory.scala b/modules/config/src/main/scala/docspell/config/ConfigFactory.scala new file mode 100644 index 00000000..27661f99 --- /dev/null +++ b/modules/config/src/main/scala/docspell/config/ConfigFactory.scala @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.config + +import scala.reflect.ClassTag + +import cats.data.OptionT +import cats.effect._ +import cats.implicits._ +import fs2.io.file.{Files, Path} + +import docspell.common.Logger + +import pureconfig.{ConfigReader, ConfigSource} + +object ConfigFactory { + + /** Reads the configuration trying the following in order: + * 1. if 'args' contains at least one element, the first is interpreted as a config + * file + * 1. otherwise check the system property 'config.file' for an existing file and use + * it if it does exist; ignore if it doesn't exist + * 1. if no file is found, read the config from environment variables falling back to + * the default config + */ + def default[F[_]: Async, C: ClassTag: ConfigReader](logger: Logger[F], atPath: String)( + args: List[String] + ): F[C] = + findFileFromArgs(args).flatMap { + case Some(file) => + logger.info(s"Using config file: $file") *> + readFile[F, C](file, atPath) + case None => + checkSystemProperty.value.flatMap { + case Some(file) => + logger.info(s"Using config file from system property: $file") *> + readConfig(atPath) + case None => + logger.info("Using config from environment variables!") *> + readEnv(atPath) + } + } + + /** Reads the configuration from the given file. */ + private def readFile[F[_]: Sync, C: ClassTag: ConfigReader]( + file: Path, + at: String + ): F[C] = + Sync[F].delay { + System.setProperty( + "config.file", + file.toNioPath.toAbsolutePath.normalize.toString + ) + ConfigSource.default.at(at).loadOrThrow[C] + } + + /** Reads the config as specified in typesafe's config library; usually loading the file + * given as system property 'config.file'. + */ + private def readConfig[F[_]: Sync, C: ClassTag: ConfigReader]( + at: String + ): F[C] = + Sync[F].delay(ConfigSource.default.at(at).loadOrThrow[C]) + + /** Reads the configuration from environment variables. */ + private def readEnv[F[_]: Sync, C: ClassTag: ConfigReader](at: String): F[C] = + Sync[F].delay(ConfigSource.fromConfig(EnvConfig.get).at(at).loadOrThrow[C]) + + /** Uses the first argument as a path to the config file. If it is specified but the + * file doesn't exist, an exception is thrown. + */ + private def findFileFromArgs[F[_]: Async](args: List[String]): F[Option[Path]] = + args.headOption + .map(Path.apply) + .traverse(p => + Files[F].exists(p).flatMap { + case true => p.pure[F] + case false => Async[F].raiseError(new Exception(s"File not found: $p")) + } + ) + + /** If the system property 'config.file' is set, it is checked whether the file exists. + * If it doesn't exist, the property is removed to not raise any exception. In contrast + * to giving the file as argument, it is not an error to specify a non-existing file + * via a system property. + */ + private def checkSystemProperty[F[_]: Async]: OptionT[F, Path] = + for { + cf <- OptionT( + Sync[F].delay( + Option(System.getProperty("config.file")).map(_.trim).filter(_.nonEmpty) + ) + ) + cp = Path(cf) + exists <- OptionT.liftF(Files[F].exists(cp)) + file <- + if (exists) OptionT.pure[F](cp) + else + OptionT + .liftF(Sync[F].delay(System.clearProperty("config.file"))) + .flatMap(_ => OptionT.none[F, Path]) + } yield file + +} diff --git a/modules/config/src/main/scala/docspell/config/EnvConfig.scala b/modules/config/src/main/scala/docspell/config/EnvConfig.scala new file mode 100644 index 00000000..37e05596 --- /dev/null +++ b/modules/config/src/main/scala/docspell/config/EnvConfig.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.config + +import java.util.Properties + +import scala.collection.{MapView, mutable} +import scala.jdk.CollectionConverters._ + +import com.typesafe.config.{Config, ConfigFactory} + +/** Creates a config from environment variables. + * + * The env variables are expected to be in same form as the config keys with the + * following mangling: a dot is replaced by an underscore character, because this is the + * standard separator for env variables. In order to represent dashes, two underscores + * are needed (and for one underscore use three underscores in the env variable). + * + * For example, the config key + * {{{ + * docspell.server.app-name + * }}} + * can be given as env variable + * {{{ + * DOCSPELL_SERVER_APP__NAME + * }}} + */ +object EnvConfig { + + /** The config from current environment. */ + lazy val get: Config = + loadFrom(System.getenv().asScala.view) + + def loadFrom(env: MapView[String, String]): Config = { + val cfg = new Properties() + for (key <- env.keySet if key.startsWith("DOCSPELL_")) + cfg.setProperty(envToProp(key), env(key)) + + ConfigFactory + .parseProperties(cfg) + .withFallback(ConfigFactory.defaultReference()) + .resolve() + } + + /** Docspell has all lowercase key names and uses snake case. + * + * So this converts to lowercase and then replaces underscores (like + * [[com.typesafe.config.ConfigFactory.systemEnvironmentOverrides()]] + * + * - 3 underscores -> `_` (underscore) + * - 2 underscores -> `-` (dash) + * - 1 underscore -> `.` (dot) + */ + private[config] def envToProp(v: String): String = { + val len = v.length + val buffer = new mutable.StringBuilder() + val underscoreMapping = Map(3 -> '_', 2 -> '-', 1 -> '.').withDefault(_ => '_') + @annotation.tailrec + def go(current: Int, underscores: Int): String = + if (current >= len) buffer.toString() + else + v.charAt(current) match { + case '_' => go(current + 1, underscores + 1) + case c => + if (underscores > 0) { + buffer.append(underscoreMapping(underscores)) + } + buffer.append(c.toLower) + go(current + 1, 0) + } + + go(0, 0) + } +} diff --git a/modules/common/src/main/scala/docspell/common/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala similarity index 97% rename from modules/common/src/main/scala/docspell/common/config/Implicits.scala rename to modules/config/src/main/scala/docspell/config/Implicits.scala index 81ff0a4c..77136cbc 100644 --- a/modules/common/src/main/scala/docspell/common/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.common.config +package docspell.config import java.nio.file.{Path => JPath} @@ -15,12 +15,11 @@ import fs2.io.file.Path import docspell.common._ import com.github.eikek.calev.CalEvent -import pureconfig._ +import pureconfig.ConfigReader import pureconfig.error.{CannotConvert, FailureReason} import scodec.bits.ByteVector object Implicits { - implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) diff --git a/modules/config/src/test/resources/reference.conf b/modules/config/src/test/resources/reference.conf new file mode 100644 index 00000000..9a1a79c8 --- /dev/null +++ b/modules/config/src/test/resources/reference.conf @@ -0,0 +1,5 @@ +docspell.server { + bind { + port = 7880 + } +} \ No newline at end of file diff --git a/modules/config/src/test/scala/docspell/config/EnvConfigTest.scala b/modules/config/src/test/scala/docspell/config/EnvConfigTest.scala new file mode 100644 index 00000000..044e0e57 --- /dev/null +++ b/modules/config/src/test/scala/docspell/config/EnvConfigTest.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.config + +import munit.FunSuite + +class EnvConfigTest extends FunSuite { + + test("convert underscores") { + assertEquals(EnvConfig.envToProp("A_B_C"), "a.b.c") + assertEquals(EnvConfig.envToProp("A_B__C"), "a.b-c") + assertEquals(EnvConfig.envToProp("AA_BB__CC___D"), "aa.bb-cc_d") + } + + test("insert docspell keys") { + val cfg = EnvConfig.loadFrom( + Map( + "DOCSPELL_SERVER_APP__NAME" -> "Hello!", + "DOCSPELL_JOEX_BIND_PORT" -> "1234" + ).view + ) + + assertEquals(cfg.getString("docspell.server.app-name"), "Hello!") + assertEquals(cfg.getInt("docspell.joex.bind.port"), 1234) + } + + test("find default values from reference.conf") { + val cfg = EnvConfig.loadFrom( + Map( + "DOCSPELL_SERVER_APP__NAME" -> "Hello!", + "DOCSPELL_JOEX_BIND_PORT" -> "1234" + ).view + ) + assertEquals(cfg.getInt("docspell.server.bind.port"), 7880) + } + + test("discard non docspell keys") { + val cfg = EnvConfig.loadFrom(Map("A_B_C" -> "12").view) + assert(!cfg.hasPath("a.b.c")) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index 24ef9322..4b3aaf4d 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -8,9 +8,12 @@ package docspell.joex import cats.data.Validated import cats.data.ValidatedNec +import cats.effect.Async import cats.implicits._ -import docspell.common.config.Implicits._ +import docspell.common.Logger +import docspell.config.ConfigFactory +import docspell.config.Implicits._ import docspell.joex.scheduler.CountingScheme import emil.MailAddress @@ -22,8 +25,12 @@ import yamusca.imports._ object ConfigFile { import Implicits._ - def loadConfig: Config = - validOrThrow(ConfigSource.default.at("docspell.joex").loadOrThrow[Config]) + def loadConfig[F[_]: Async](args: List[String]): F[Config] = { + val logger = Logger.log4s[F](org.log4s.getLogger) + ConfigFactory + .default[F, Config](logger, "docspell.joex")(args) + .map(cfg => validOrThrow(cfg)) + } private def validOrThrow(cfg: Config): Config = validate(cfg).fold(err => sys.error(err.toList.mkString("- ", "\n", "")), identity) diff --git a/modules/joex/src/main/scala/docspell/joex/Main.scala b/modules/joex/src/main/scala/docspell/joex/Main.scala index fa20b0e5..6f96e1e0 100644 --- a/modules/joex/src/main/scala/docspell/joex/Main.scala +++ b/modules/joex/src/main/scala/docspell/joex/Main.scala @@ -6,67 +6,45 @@ package docspell.joex -import java.nio.file.{Files, Paths} - import cats.effect._ -import cats.implicits._ import docspell.common._ -import org.log4s._ +import org.log4s.getLogger object Main extends IOApp { - private[this] val logger = getLogger - val blockingEC = - ThreadFactories.cached[IO](ThreadFactories.ofName("docspell-joex-blocking")) - val connectEC = + private val logger: Logger[IO] = Logger.log4s[IO](getLogger) + + private val connectEC = ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-joex-dbconnect")) - val restserverEC = - ThreadFactories.workSteal[IO](ThreadFactories.ofNameFJ("docspell-joex-server")) - def run(args: List[String]) = { - args match { - case file :: Nil => - val path = Paths.get(file).toAbsolutePath.normalize - logger.info(s"Using given config file: $path") - System.setProperty("config.file", file) - case _ => - Option(System.getProperty("config.file")) match { - case Some(f) if f.nonEmpty => - val path = Paths.get(f).toAbsolutePath.normalize - if (!Files.exists(path)) { - logger.info(s"Not using config file '$f' because it doesn't exist") - System.clearProperty("config.file") - } else - logger.info(s"Using config file from system properties: $f") - case _ => - } - } + def run(args: List[String]): IO[ExitCode] = + for { + cfg <- ConfigFile.loadConfig[IO](args) + banner = Banner( + "JOEX", + BuildInfo.version, + BuildInfo.gitHeadCommit, + cfg.jdbc.url, + Option(System.getProperty("config.file")), + cfg.appId, + cfg.baseUrl, + Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled) + ) + _ <- logger.info(s"\n${banner.render("***>")}") + _ <- + if (EnvMode.current.isDev) { + logger.warn(">>>>> Docspell is running in DEV mode! <<<<<") + } else IO(()) - val cfg = ConfigFile.loadConfig - val banner = Banner( - "JOEX", - BuildInfo.version, - BuildInfo.gitHeadCommit, - cfg.jdbc.url, - Option(System.getProperty("config.file")), - cfg.appId, - cfg.baseUrl, - Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled) - ) - logger.info(s"\n${banner.render("***>")}") - if (EnvMode.current.isDev) { - logger.warn(">>>>> Docspell is running in DEV mode! <<<<<") - } - - val pools = connectEC.map(Pools.apply) - pools.use(p => - JoexServer - .stream[IO](cfg, p) - .compile - .drain - .as(ExitCode.Success) - ) - } + pools = connectEC.map(Pools.apply) + rc <- pools.use(p => + JoexServer + .stream[IO](cfg, p) + .compile + .drain + .as(ExitCode.Success) + ) + } yield rc } diff --git a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala index c321ee71..7bebe31e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala @@ -8,10 +8,13 @@ package docspell.restserver import cats.Semigroup import cats.data.{Validated, ValidatedNec} +import cats.effect.Async import cats.implicits._ import docspell.backend.signup.{Config => SignupConfig} -import docspell.common.config.Implicits._ +import docspell.common.Logger +import docspell.config.ConfigFactory +import docspell.config.Implicits._ import docspell.oidc.{ProviderConfig, SignatureAlgo} import docspell.restserver.auth.OpenId @@ -21,8 +24,12 @@ import pureconfig.generic.auto._ object ConfigFile { import Implicits._ - def loadConfig: Config = - Validate(ConfigSource.default.at("docspell.server").loadOrThrow[Config]) + def loadConfig[F[_]: Async](args: List[String]): F[Config] = { + val logger = Logger.log4s(org.log4s.getLogger) + ConfigFactory + .default[F, Config](logger, "docspell.server")(args) + .map(cfg => Validate(cfg)) + } object Implicits { implicit val signupModeReader: ConfigReader[SignupConfig.Mode] = diff --git a/modules/restserver/src/main/scala/docspell/restserver/Main.scala b/modules/restserver/src/main/scala/docspell/restserver/Main.scala index 7d441783..5907ff41 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Main.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Main.scala @@ -6,46 +6,21 @@ package docspell.restserver -import java.nio.file.{Files, Paths} - import cats.effect._ -import cats.implicits._ import docspell.common._ -import org.log4s._ +import org.log4s.getLogger object Main extends IOApp { - private[this] val logger = getLogger + private[this] val logger: Logger[IO] = Logger.log4s(getLogger) - val blockingEC = - ThreadFactories.cached[IO](ThreadFactories.ofName("docspell-restserver-blocking")) - val connectEC = + private val connectEC = ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-dbconnect")) - val restserverEC = - ThreadFactories.workSteal[IO](ThreadFactories.ofNameFJ("docspell-restserver")) - def run(args: List[String]) = { - args match { - case file :: Nil => - val path = Paths.get(file).toAbsolutePath.normalize - logger.info(s"Using given config file: $path") - System.setProperty("config.file", file) - case _ => - Option(System.getProperty("config.file")) match { - case Some(f) if f.nonEmpty => - val path = Paths.get(f).toAbsolutePath.normalize - if (!Files.exists(path)) { - logger.info(s"Not using config file '$f' because it doesn't exist") - System.clearProperty("config.file") - } else - logger.info(s"Using config file from system properties: $f") - case _ => - } - } - - val cfg = ConfigFile.loadConfig - val banner = Banner( + def run(args: List[String]) = for { + cfg <- ConfigFile.loadConfig[IO](args) + banner = Banner( "REST Server", BuildInfo.version, BuildInfo.gitHeadCommit, @@ -55,18 +30,20 @@ object Main extends IOApp { cfg.baseUrl, Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled) ) - val pools = connectEC.map(Pools.apply) - logger.info(s"\n${banner.render("***>")}") - if (EnvMode.current.isDev) { - logger.warn(">>>>> Docspell is running in DEV mode! <<<<<") - } + _ <- logger.info(s"\n${banner.render("***>")}") + _ <- + if (EnvMode.current.isDev) { + logger.warn(">>>>> Docspell is running in DEV mode! <<<<<") + } else IO(()) - pools.use(p => - RestServer - .stream[IO](cfg, p) - .compile - .drain - .as(ExitCode.Success) - ) - } + pools = connectEC.map(Pools.apply) + rc <- + pools.use(p => + RestServer + .stream[IO](cfg, p) + .compile + .drain + .as(ExitCode.Success) + ) + } yield rc }