Refactor config loading and add config from environment

Issue: #1121

.scalafmt.conf

@@ -2,6 +2,7 @@ version = "3.0.6"
 
 preset = default
 align.preset = some
+runner.dialect = scala213
 
 maxColumn = 90
 

build.sbt

@@ -293,10 +293,23 @@ val common = project
         Dependencies.circe ++
         Dependencies.loggingApi ++
         Dependencies.calevCore ++
-        Dependencies.calevCirce ++
+        Dependencies.calevCirce
-        Dependencies.pureconfig.map(_ % "optional")
   )
 
+val config = project
+  .in(file("modules/config"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .settings(testSettingsMUnit)
+  .settings(
+    name := "docspell-config",
+    addCompilerPlugin(Dependencies.kindProjectorPlugin),
+    libraryDependencies ++=
+      Dependencies.fs2 ++
+        Dependencies.pureconfig
+  )
+  .dependsOn(common)
+
 // Some example files for testing
 // https://file-examples.com/index.php/sample-documents-download/sample-doc-download/
 val files = project
@@ -603,7 +616,17 @@ val joex = project
     ),
     Revolver.enableDebugging(port = 5051, suspend = false)
   )
-  .dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr)
+  .dependsOn(
+    config,
+    store,
+    backend,
+    extract,
+    convert,
+    analysis,
+    joexapi,
+    restapi,
+    ftssolr
+  )
 
 val restserver = project
   .in(file("modules/restserver"))
@@ -666,7 +689,7 @@ val restserver = project
       }
     }
   )
-  .dependsOn(restapi, joexapi, backend, webapp, ftssolr, oidc)
+  .dependsOn(config, restapi, joexapi, backend, webapp, ftssolr, oidc)
 
 // --- Website Documentation
 
@@ -731,6 +754,7 @@ val root = project
   )
   .aggregate(
     common,
+    config,
     extract,
     convert,
     analysis,

modules/config/src/main/scala/docspell/config/ConfigFactory.scala

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import scala.reflect.ClassTag
+
+import cats.data.OptionT
+import cats.effect._
+import cats.implicits._
+import fs2.io.file.{Files, Path}
+
+import docspell.common.Logger
+
+import pureconfig.{ConfigReader, ConfigSource}
+
+object ConfigFactory {
+
+  /** Reads the configuration trying the following in order:
+    *   1. if 'args' contains at least one element, the first is interpreted as a config
+    *      file
+    *   1. otherwise check the system property 'config.file' for an existing file and use
+    *      it if it does exist; ignore if it doesn't exist
+    *   1. if no file is found, read the config from environment variables falling back to
+    *      the default config
+    */
+  def default[F[_]: Async, C: ClassTag: ConfigReader](logger: Logger[F], atPath: String)(
+      args: List[String]
+  ): F[C] =
+    findFileFromArgs(args).flatMap {
+      case Some(file) =>
+        logger.info(s"Using config file: $file") *>
+          readFile[F, C](file, atPath)
+      case None =>
+        checkSystemProperty.value.flatMap {
+          case Some(file) =>
+            logger.info(s"Using config file from system property: $file") *>
+              readConfig(atPath)
+          case None =>
+            logger.info("Using config from environment variables!") *>
+              readEnv(atPath)
+        }
+    }
+
+  /** Reads the configuration from the given file. */
+  private def readFile[F[_]: Sync, C: ClassTag: ConfigReader](
+      file: Path,
+      at: String
+  ): F[C] =
+    Sync[F].delay {
+      System.setProperty(
+        "config.file",
+        file.toNioPath.toAbsolutePath.normalize.toString
+      )
+      ConfigSource.default.at(at).loadOrThrow[C]
+    }
+
+  /** Reads the config as specified in typesafe's config library; usually loading the file
+    * given as system property 'config.file'.
+    */
+  private def readConfig[F[_]: Sync, C: ClassTag: ConfigReader](
+      at: String
+  ): F[C] =
+    Sync[F].delay(ConfigSource.default.at(at).loadOrThrow[C])
+
+  /** Reads the configuration from environment variables. */
+  private def readEnv[F[_]: Sync, C: ClassTag: ConfigReader](at: String): F[C] =
+    Sync[F].delay(ConfigSource.fromConfig(EnvConfig.get).at(at).loadOrThrow[C])
+
+  /** Uses the first argument as a path to the config file. If it is specified but the
+    * file doesn't exist, an exception is thrown.
+    */
+  private def findFileFromArgs[F[_]: Async](args: List[String]): F[Option[Path]] =
+    args.headOption
+      .map(Path.apply)
+      .traverse(p =>
+        Files[F].exists(p).flatMap {
+          case true  => p.pure[F]
+          case false => Async[F].raiseError(new Exception(s"File not found: $p"))
+        }
+      )
+
+  /** If the system property 'config.file' is set, it is checked whether the file exists.
+    * If it doesn't exist, the property is removed to not raise any exception. In contrast
+    * to giving the file as argument, it is not an error to specify a non-existing file
+    * via a system property.
+    */
+  private def checkSystemProperty[F[_]: Async]: OptionT[F, Path] =
+    for {
+      cf <- OptionT(
+        Sync[F].delay(
+          Option(System.getProperty("config.file")).map(_.trim).filter(_.nonEmpty)
+        )
+      )
+      cp = Path(cf)
+      exists <- OptionT.liftF(Files[F].exists(cp))
+      file <-
+        if (exists) OptionT.pure[F](cp)
+        else
+          OptionT
+            .liftF(Sync[F].delay(System.clearProperty("config.file")))
+            .flatMap(_ => OptionT.none[F, Path])
+    } yield file
+
+}

modules/config/src/main/scala/docspell/config/EnvConfig.scala

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import java.util.Properties
+
+import scala.collection.{MapView, mutable}
+import scala.jdk.CollectionConverters._
+
+import com.typesafe.config.{Config, ConfigFactory}
+
+/** Creates a config from environment variables.
+  *
+  * The env variables are expected to be in same form as the config keys with the
+  * following mangling: a dot is replaced by an underscore character, because this is the
+  * standard separator for env variables. In order to represent dashes, two underscores
+  * are needed (and for one underscore use three underscores in the env variable).
+  *
+  * For example, the config key
+  * {{{
+  *   docspell.server.app-name
+  * }}}
+  * can be given as env variable
+  * {{{
+  *   DOCSPELL_SERVER_APP__NAME
+  * }}}
+  */
+object EnvConfig {
+
+  /** The config from current environment. */
+  lazy val get: Config =
+    loadFrom(System.getenv().asScala.view)
+
+  def loadFrom(env: MapView[String, String]): Config = {
+    val cfg = new Properties()
+    for (key <- env.keySet if key.startsWith("DOCSPELL_"))
+      cfg.setProperty(envToProp(key), env(key))
+
+    ConfigFactory
+      .parseProperties(cfg)
+      .withFallback(ConfigFactory.defaultReference())
+      .resolve()
+  }
+
+  /** Docspell has all lowercase key names and uses snake case.
+    *
+    * So this converts to lowercase and then replaces underscores (like
+    * [[com.typesafe.config.ConfigFactory.systemEnvironmentOverrides()]]
+    *
+    *   - 3 underscores -> `_` (underscore)
+    *   - 2 underscores -> `-` (dash)
+    *   - 1 underscore -> `.` (dot)
+    */
+  private[config] def envToProp(v: String): String = {
+    val len = v.length
+    val buffer = new mutable.StringBuilder()
+    val underscoreMapping = Map(3 -> '_', 2 -> '-', 1 -> '.').withDefault(_ => '_')
+    @annotation.tailrec
+    def go(current: Int, underscores: Int): String =
+      if (current >= len) buffer.toString()
+      else
+        v.charAt(current) match {
+          case '_' => go(current + 1, underscores + 1)
+          case c =>
+            if (underscores > 0) {
+              buffer.append(underscoreMapping(underscores))
+            }
+            buffer.append(c.toLower)
+            go(current + 1, 0)
+        }
+
+    go(0, 0)
+  }
+}

modules/config/src/main/scala/docspell/config/Implicits.scala

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-package docspell.common.config
+package docspell.config
 
 import java.nio.file.{Path => JPath}
 
@@ -15,12 +15,11 @@ import fs2.io.file.Path
 import docspell.common._
 
 import com.github.eikek.calev.CalEvent
-import pureconfig._
+import pureconfig.ConfigReader
 import pureconfig.error.{CannotConvert, FailureReason}
 import scodec.bits.ByteVector
 
 object Implicits {
-
   implicit val accountIdReader: ConfigReader[AccountId] =
     ConfigReader[String].emap(reason(AccountId.parse))
 

modules/config/src/test/resources/reference.conf

@@ -0,0 +1,5 @@
+docspell.server {
+  bind {
+    port = 7880
+  }
+}

modules/config/src/test/scala/docspell/config/EnvConfigTest.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import munit.FunSuite
+
+class EnvConfigTest extends FunSuite {
+
+  test("convert underscores") {
+    assertEquals(EnvConfig.envToProp("A_B_C"), "a.b.c")
+    assertEquals(EnvConfig.envToProp("A_B__C"), "a.b-c")
+    assertEquals(EnvConfig.envToProp("AA_BB__CC___D"), "aa.bb-cc_d")
+  }
+
+  test("insert docspell keys") {
+    val cfg = EnvConfig.loadFrom(
+      Map(
+        "DOCSPELL_SERVER_APP__NAME" -> "Hello!",
+        "DOCSPELL_JOEX_BIND_PORT" -> "1234"
+      ).view
+    )
+
+    assertEquals(cfg.getString("docspell.server.app-name"), "Hello!")
+    assertEquals(cfg.getInt("docspell.joex.bind.port"), 1234)
+  }
+
+  test("find default values from reference.conf") {
+    val cfg = EnvConfig.loadFrom(
+      Map(
+        "DOCSPELL_SERVER_APP__NAME" -> "Hello!",
+        "DOCSPELL_JOEX_BIND_PORT" -> "1234"
+      ).view
+    )
+    assertEquals(cfg.getInt("docspell.server.bind.port"), 7880)
+  }
+
+  test("discard non docspell keys") {
+    val cfg = EnvConfig.loadFrom(Map("A_B_C" -> "12").view)
+    assert(!cfg.hasPath("a.b.c"))
+  }
+}

modules/joex/src/main/scala/docspell/joex/ConfigFile.scala

@@ -8,9 +8,12 @@ package docspell.joex
 
 import cats.data.Validated
 import cats.data.ValidatedNec
+import cats.effect.Async
 import cats.implicits._
 
-import docspell.common.config.Implicits._
+import docspell.common.Logger
+import docspell.config.ConfigFactory
+import docspell.config.Implicits._
 import docspell.joex.scheduler.CountingScheme
 
 import emil.MailAddress
@@ -22,8 +25,12 @@ import yamusca.imports._
 object ConfigFile {
   import Implicits._
 
-  def loadConfig: Config =
+  def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
-    validOrThrow(ConfigSource.default.at("docspell.joex").loadOrThrow[Config])
+    val logger = Logger.log4s[F](org.log4s.getLogger)
+    ConfigFactory
+      .default[F, Config](logger, "docspell.joex")(args)
+      .map(cfg => validOrThrow(cfg))
+  }
 
   private def validOrThrow(cfg: Config): Config =
     validate(cfg).fold(err => sys.error(err.toList.mkString("- ", "\n", "")), identity)

modules/joex/src/main/scala/docspell/joex/Main.scala

@@ -6,46 +6,23 @@
 
 package docspell.joex
 
-import java.nio.file.{Files, Paths}
-
 import cats.effect._
-import cats.implicits._
 
 import docspell.common._
 
-import org.log4s._
+import org.log4s.getLogger
 
 object Main extends IOApp {
-  private[this] val logger = getLogger
 
-  val blockingEC =
+  private val logger: Logger[IO] = Logger.log4s[IO](getLogger)
-    ThreadFactories.cached[IO](ThreadFactories.ofName("docspell-joex-blocking"))
+
-  val connectEC =
+  private val connectEC =
     ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-joex-dbconnect"))
-  val restserverEC =
-    ThreadFactories.workSteal[IO](ThreadFactories.ofNameFJ("docspell-joex-server"))
 
-  def run(args: List[String]) = {
+  def run(args: List[String]): IO[ExitCode] =
-    args match {
+    for {
-      case file :: Nil =>
+      cfg <- ConfigFile.loadConfig[IO](args)
-        val path = Paths.get(file).toAbsolutePath.normalize
+      banner = Banner(
-        logger.info(s"Using given config file: $path")
-        System.setProperty("config.file", file)
-      case _ =>
-        Option(System.getProperty("config.file")) match {
-          case Some(f) if f.nonEmpty =>
-            val path = Paths.get(f).toAbsolutePath.normalize
-            if (!Files.exists(path)) {
-              logger.info(s"Not using config file '$f' because it doesn't exist")
-              System.clearProperty("config.file")
-            } else
-              logger.info(s"Using config file from system properties: $f")
-          case _ =>
-        }
-    }
-
-    val cfg = ConfigFile.loadConfig
-    val banner = Banner(
         "JOEX",
         BuildInfo.version,
         BuildInfo.gitHeadCommit,
@@ -55,18 +32,19 @@ object Main extends IOApp {
         cfg.baseUrl,
         Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled)
       )
-    logger.info(s"\n${banner.render("***>")}")
+      _ <- logger.info(s"\n${banner.render("***>")}")
+      _ <-
         if (EnvMode.current.isDev) {
           logger.warn(">>>>>   Docspell is running in DEV mode!   <<<<<")
-    }
+        } else IO(())
 
-    val pools = connectEC.map(Pools.apply)
+      pools = connectEC.map(Pools.apply)
-    pools.use(p =>
+      rc <- pools.use(p =>
         JoexServer
           .stream[IO](cfg, p)
           .compile
           .drain
           .as(ExitCode.Success)
       )
-  }
+    } yield rc
 }

modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala

@@ -8,10 +8,13 @@ package docspell.restserver
 
 import cats.Semigroup
 import cats.data.{Validated, ValidatedNec}
+import cats.effect.Async
 import cats.implicits._
 
 import docspell.backend.signup.{Config => SignupConfig}
-import docspell.common.config.Implicits._
+import docspell.common.Logger
+import docspell.config.ConfigFactory
+import docspell.config.Implicits._
 import docspell.oidc.{ProviderConfig, SignatureAlgo}
 import docspell.restserver.auth.OpenId
 
@@ -21,8 +24,12 @@ import pureconfig.generic.auto._
 object ConfigFile {
   import Implicits._
 
-  def loadConfig: Config =
+  def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
-    Validate(ConfigSource.default.at("docspell.server").loadOrThrow[Config])
+    val logger = Logger.log4s(org.log4s.getLogger)
+    ConfigFactory
+      .default[F, Config](logger, "docspell.server")(args)
+      .map(cfg => Validate(cfg))
+  }
 
   object Implicits {
     implicit val signupModeReader: ConfigReader[SignupConfig.Mode] =

modules/restserver/src/main/scala/docspell/restserver/Main.scala

@@ -6,46 +6,21 @@
 
 package docspell.restserver
 
-import java.nio.file.{Files, Paths}
-
 import cats.effect._
-import cats.implicits._
 
 import docspell.common._
 
-import org.log4s._
+import org.log4s.getLogger
 
 object Main extends IOApp {
-  private[this] val logger = getLogger
+  private[this] val logger: Logger[IO] = Logger.log4s(getLogger)
 
-  val blockingEC =
+  private val connectEC =
-    ThreadFactories.cached[IO](ThreadFactories.ofName("docspell-restserver-blocking"))
-  val connectEC =
     ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-dbconnect"))
-  val restserverEC =
-    ThreadFactories.workSteal[IO](ThreadFactories.ofNameFJ("docspell-restserver"))
 
-  def run(args: List[String]) = {
+  def run(args: List[String]) = for {
-    args match {
+    cfg <- ConfigFile.loadConfig[IO](args)
-      case file :: Nil =>
+    banner = Banner(
-        val path = Paths.get(file).toAbsolutePath.normalize
-        logger.info(s"Using given config file: $path")
-        System.setProperty("config.file", file)
-      case _ =>
-        Option(System.getProperty("config.file")) match {
-          case Some(f) if f.nonEmpty =>
-            val path = Paths.get(f).toAbsolutePath.normalize
-            if (!Files.exists(path)) {
-              logger.info(s"Not using config file '$f' because it doesn't exist")
-              System.clearProperty("config.file")
-            } else
-              logger.info(s"Using config file from system properties: $f")
-          case _ =>
-        }
-    }
-
-    val cfg = ConfigFile.loadConfig
-    val banner = Banner(
       "REST Server",
       BuildInfo.version,
       BuildInfo.gitHeadCommit,
@@ -55,12 +30,14 @@ object Main extends IOApp {
       cfg.baseUrl,
       Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled)
     )
-    val pools = connectEC.map(Pools.apply)
+    _ <- logger.info(s"\n${banner.render("***>")}")
-    logger.info(s"\n${banner.render("***>")}")
+    _ <-
       if (EnvMode.current.isDev) {
         logger.warn(">>>>>   Docspell is running in DEV mode!   <<<<<")
-    }
+      } else IO(())
 
+    pools = connectEC.map(Pools.apply)
+    rc <-
       pools.use(p =>
         RestServer
           .stream[IO](cfg, p)
@@ -68,5 +45,5 @@ object Main extends IOApp {
           .drain
           .as(ExitCode.Success)
       )
-  }
+  } yield rc
 }