Refactor config loading and add config from environment

Issue: #1121

modules/config/src/main/scala/docspell/config/ConfigFactory.scala

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import scala.reflect.ClassTag
+
+import cats.data.OptionT
+import cats.effect._
+import cats.implicits._
+import fs2.io.file.{Files, Path}
+
+import docspell.common.Logger
+
+import pureconfig.{ConfigReader, ConfigSource}
+
+object ConfigFactory {
+
+  /** Reads the configuration trying the following in order:
+    *   1. if 'args' contains at least one element, the first is interpreted as a config
+    *      file
+    *   1. otherwise check the system property 'config.file' for an existing file and use
+    *      it if it does exist; ignore if it doesn't exist
+    *   1. if no file is found, read the config from environment variables falling back to
+    *      the default config
+    */
+  def default[F[_]: Async, C: ClassTag: ConfigReader](logger: Logger[F], atPath: String)(
+      args: List[String]
+  ): F[C] =
+    findFileFromArgs(args).flatMap {
+      case Some(file) =>
+        logger.info(s"Using config file: $file") *>
+          readFile[F, C](file, atPath)
+      case None =>
+        checkSystemProperty.value.flatMap {
+          case Some(file) =>
+            logger.info(s"Using config file from system property: $file") *>
+              readConfig(atPath)
+          case None =>
+            logger.info("Using config from environment variables!") *>
+              readEnv(atPath)
+        }
+    }
+
+  /** Reads the configuration from the given file. */
+  private def readFile[F[_]: Sync, C: ClassTag: ConfigReader](
+      file: Path,
+      at: String
+  ): F[C] =
+    Sync[F].delay {
+      System.setProperty(
+        "config.file",
+        file.toNioPath.toAbsolutePath.normalize.toString
+      )
+      ConfigSource.default.at(at).loadOrThrow[C]
+    }
+
+  /** Reads the config as specified in typesafe's config library; usually loading the file
+    * given as system property 'config.file'.
+    */
+  private def readConfig[F[_]: Sync, C: ClassTag: ConfigReader](
+      at: String
+  ): F[C] =
+    Sync[F].delay(ConfigSource.default.at(at).loadOrThrow[C])
+
+  /** Reads the configuration from environment variables. */
+  private def readEnv[F[_]: Sync, C: ClassTag: ConfigReader](at: String): F[C] =
+    Sync[F].delay(ConfigSource.fromConfig(EnvConfig.get).at(at).loadOrThrow[C])
+
+  /** Uses the first argument as a path to the config file. If it is specified but the
+    * file doesn't exist, an exception is thrown.
+    */
+  private def findFileFromArgs[F[_]: Async](args: List[String]): F[Option[Path]] =
+    args.headOption
+      .map(Path.apply)
+      .traverse(p =>
+        Files[F].exists(p).flatMap {
+          case true  => p.pure[F]
+          case false => Async[F].raiseError(new Exception(s"File not found: $p"))
+        }
+      )
+
+  /** If the system property 'config.file' is set, it is checked whether the file exists.
+    * If it doesn't exist, the property is removed to not raise any exception. In contrast
+    * to giving the file as argument, it is not an error to specify a non-existing file
+    * via a system property.
+    */
+  private def checkSystemProperty[F[_]: Async]: OptionT[F, Path] =
+    for {
+      cf <- OptionT(
+        Sync[F].delay(
+          Option(System.getProperty("config.file")).map(_.trim).filter(_.nonEmpty)
+        )
+      )
+      cp = Path(cf)
+      exists <- OptionT.liftF(Files[F].exists(cp))
+      file <-
+        if (exists) OptionT.pure[F](cp)
+        else
+          OptionT
+            .liftF(Sync[F].delay(System.clearProperty("config.file")))
+            .flatMap(_ => OptionT.none[F, Path])
+    } yield file
+
+}

modules/config/src/main/scala/docspell/config/EnvConfig.scala

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import java.util.Properties
+
+import scala.collection.{MapView, mutable}
+import scala.jdk.CollectionConverters._
+
+import com.typesafe.config.{Config, ConfigFactory}
+
+/** Creates a config from environment variables.
+  *
+  * The env variables are expected to be in same form as the config keys with the
+  * following mangling: a dot is replaced by an underscore character, because this is the
+  * standard separator for env variables. In order to represent dashes, two underscores
+  * are needed (and for one underscore use three underscores in the env variable).
+  *
+  * For example, the config key
+  * {{{
+  *   docspell.server.app-name
+  * }}}
+  * can be given as env variable
+  * {{{
+  *   DOCSPELL_SERVER_APP__NAME
+  * }}}
+  */
+object EnvConfig {
+
+  /** The config from current environment. */
+  lazy val get: Config =
+    loadFrom(System.getenv().asScala.view)
+
+  def loadFrom(env: MapView[String, String]): Config = {
+    val cfg = new Properties()
+    for (key <- env.keySet if key.startsWith("DOCSPELL_"))
+      cfg.setProperty(envToProp(key), env(key))
+
+    ConfigFactory
+      .parseProperties(cfg)
+      .withFallback(ConfigFactory.defaultReference())
+      .resolve()
+  }
+
+  /** Docspell has all lowercase key names and uses snake case.
+    *
+    * So this converts to lowercase and then replaces underscores (like
+    * [[com.typesafe.config.ConfigFactory.systemEnvironmentOverrides()]]
+    *
+    *   - 3 underscores -> `_` (underscore)
+    *   - 2 underscores -> `-` (dash)
+    *   - 1 underscore -> `.` (dot)
+    */
+  private[config] def envToProp(v: String): String = {
+    val len = v.length
+    val buffer = new mutable.StringBuilder()
+    val underscoreMapping = Map(3 -> '_', 2 -> '-', 1 -> '.').withDefault(_ => '_')
+    @annotation.tailrec
+    def go(current: Int, underscores: Int): String =
+      if (current >= len) buffer.toString()
+      else
+        v.charAt(current) match {
+          case '_' => go(current + 1, underscores + 1)
+          case c =>
+            if (underscores > 0) {
+              buffer.append(underscoreMapping(underscores))
+            }
+            buffer.append(c.toLower)
+            go(current + 1, 0)
+        }
+
+    go(0, 0)
+  }
+}

modules/config/src/main/scala/docspell/config/Implicits.scala

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import java.nio.file.{Path => JPath}
+
+import scala.reflect.ClassTag
+
+import fs2.io.file.Path
+
+import docspell.common._
+
+import com.github.eikek.calev.CalEvent
+import pureconfig.ConfigReader
+import pureconfig.error.{CannotConvert, FailureReason}
+import scodec.bits.ByteVector
+
+object Implicits {
+  implicit val accountIdReader: ConfigReader[AccountId] =
+    ConfigReader[String].emap(reason(AccountId.parse))
+
+  implicit val pathReader: ConfigReader[Path] =
+    ConfigReader[JPath].map(Path.fromNioPath)
+
+  implicit val lenientUriReader: ConfigReader[LenientUri] =
+    ConfigReader[String].emap(reason(LenientUri.parse))
+
+  implicit val durationReader: ConfigReader[Duration] =
+    ConfigReader[scala.concurrent.duration.Duration].map(sd => Duration(sd))
+
+  implicit val passwordReader: ConfigReader[Password] =
+    ConfigReader[String].map(Password(_))
+
+  implicit val mimeTypeReader: ConfigReader[MimeType] =
+    ConfigReader[String].emap(reason(MimeType.parse))
+
+  implicit val identReader: ConfigReader[Ident] =
+    ConfigReader[String].emap(reason(Ident.fromString))
+
+  implicit val byteVectorReader: ConfigReader[ByteVector] =
+    ConfigReader[String].emap(reason { str =>
+      if (str.startsWith("hex:"))
+        ByteVector.fromHex(str.drop(4)).toRight("Invalid hex value.")
+      else if (str.startsWith("b64:"))
+        ByteVector.fromBase64(str.drop(4)).toRight("Invalid Base64 string.")
+      else
+        ByteVector
+          .encodeUtf8(str)
+          .left
+          .map(ex => s"Invalid utf8 string: ${ex.getMessage}")
+    })
+
+  implicit val caleventReader: ConfigReader[CalEvent] =
+    ConfigReader[String].emap(reason(CalEvent.parse))
+
+  implicit val priorityReader: ConfigReader[Priority] =
+    ConfigReader[String].emap(reason(Priority.fromString))
+
+  implicit val nlpModeReader: ConfigReader[NlpMode] =
+    ConfigReader[String].emap(reason(NlpMode.fromString))
+
+  def reason[A: ClassTag](
+      f: String => Either[String, A]
+  ): String => Either[FailureReason, A] =
+    in =>
+      f(in).left.map(str =>
+        CannotConvert(in, implicitly[ClassTag[A]].runtimeClass.toString, str)
+      )
+}

modules/config/src/test/resources/reference.conf

@@ -0,0 +1,5 @@
+docspell.server {
+  bind {
+    port = 7880
+  }
+}

modules/config/src/test/scala/docspell/config/EnvConfigTest.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import munit.FunSuite
+
+class EnvConfigTest extends FunSuite {
+
+  test("convert underscores") {
+    assertEquals(EnvConfig.envToProp("A_B_C"), "a.b.c")
+    assertEquals(EnvConfig.envToProp("A_B__C"), "a.b-c")
+    assertEquals(EnvConfig.envToProp("AA_BB__CC___D"), "aa.bb-cc_d")
+  }
+
+  test("insert docspell keys") {
+    val cfg = EnvConfig.loadFrom(
+      Map(
+        "DOCSPELL_SERVER_APP__NAME" -> "Hello!",
+        "DOCSPELL_JOEX_BIND_PORT" -> "1234"
+      ).view
+    )
+
+    assertEquals(cfg.getString("docspell.server.app-name"), "Hello!")
+    assertEquals(cfg.getInt("docspell.joex.bind.port"), 1234)
+  }
+
+  test("find default values from reference.conf") {
+    val cfg = EnvConfig.loadFrom(
+      Map(
+        "DOCSPELL_SERVER_APP__NAME" -> "Hello!",
+        "DOCSPELL_JOEX_BIND_PORT" -> "1234"
+      ).view
+    )
+    assertEquals(cfg.getInt("docspell.server.bind.port"), 7880)
+  }
+
+  test("discard non docspell keys") {
+    val cfg = EnvConfig.loadFrom(Map("A_B_C" -> "12").view)
+    assert(!cfg.hasPath("a.b.c"))
+  }
+}