Experiment with addons

Addons allow to execute external programs in some context inside
docspell. Currently it is possible to run them after processing files.
Addons are provided by URLs to zip files.

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.Zip
+
+final case class AddonArchive(url: LenientUri, name: String, version: String) {
+  def nameAndVersion: String =
+    s"$name-$version"
+
+  def extractTo[F[_]: Async](
+      reader: UrlReader[F],
+      directory: Path,
+      withSubdir: Boolean = true,
+      glob: Glob = Glob.all
+  ): F[Path] = {
+    val logger = docspell.logging.getLogger[F]
+    val target =
+      if (withSubdir) directory.absolute / nameAndVersion
+      else directory.absolute
+
+    Files[F]
+      .exists(target)
+      .flatMap {
+        case true => target.pure[F]
+        case false =>
+          Files[F].createDirectories(target) *>
+            reader(url)
+              .through(Zip.unzip(8192, glob))
+              .through(Zip.saveTo(logger, target, moveUp = true))
+              .compile
+              .drain
+              .as(target)
+      }
+  }
+
+  /** Read meta either from the given directory or extract the url to find the metadata
+    * file to read
+    */
+  def readMeta[F[_]: Async](
+      urlReader: UrlReader[F],
+      directory: Option[Path] = None
+  ): F[AddonMeta] =
+    directory
+      .map(AddonMeta.findInDirectory[F])
+      .getOrElse(AddonMeta.findInZip(urlReader(url)))
+}
+
+object AddonArchive {
+  def read[F[_]: Async](
+      url: LenientUri,
+      urlReader: UrlReader[F],
+      extractDir: Option[Path] = None
+  ): F[AddonArchive] = {
+    val addon = AddonArchive(url, "", "")
+    addon
+      .readMeta(urlReader, extractDir)
+      .map(m => addon.copy(name = m.meta.name, version = m.meta.version))
+  }
+
+  def dockerAndFlakeExists[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[(Boolean, Boolean)] = {
+    val files = Files[F]
+    def forPath(path: Path): F[(Boolean, Boolean)] =
+      (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled
+
+    def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] =
+      data
+        .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix")))
+        .collect {
+          case bin if bin.name == "Dockerfile" => (true, false)
+          case bin if bin.name == "flake.nix"  => (false, true)
+        }
+        .compile
+        .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2))
+
+    archive.fold(forPath, forZip)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+import cats.syntax.all._
+
+case class AddonExecutionResult(
+    addonResults: List[AddonResult],
+    pure: Boolean
+) {
+  def addonResult: AddonResult = addonResults.combineAll
+  def isFailure: Boolean = addonResult.isFailure
+  def isSuccess: Boolean = addonResult.isSuccess
+}
+
+object AddonExecutionResult {
+  val empty: AddonExecutionResult =
+    AddonExecutionResult(Nil, false)
+
+  def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult =
+    AddonExecutionResult(
+      a.addonResults ::: b.addonResults,
+      a.pure && b.pure
+    )
+
+  implicit val executionResultMonoid: Monoid[AddonExecutionResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.Kleisli
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file._
+
+import docspell.common.UrlReader
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonExecutor[F[_]] {
+
+  def config: AddonExecutorConfig
+
+  def execute(logger: Logger[F]): AddonExec[F]
+
+  def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] =
+    execute(logger).run(in)
+}
+
+object AddonExecutor {
+
+  def apply[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      urlReader: UrlReader[F]
+  ): AddonExecutor[F] =
+    new AddonExecutor[F] with AddonLoggerExtension {
+      val config = cfg
+
+      def execute(logger: Logger[F]): AddonExec[F] =
+        Kleisli { in =>
+          for {
+            _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}")
+            ctx <- prepareDirectory(
+              logger,
+              in.baseDir,
+              in.outputDir,
+              in.cacheDir,
+              in.addons
+            )
+            rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c))
+            pure = ctx.foldl(true)((b, c) => b && c.meta.isPure)
+          } yield AddonExecutionResult(rs, pure)
+        }
+
+      private def prepareDirectory(
+          logger: Logger[F],
+          baseDir: Path,
+          outDir: Path,
+          cacheDir: Path,
+          addons: List[AddonRef]
+      ): F[List[Context]] =
+        for {
+          addonsDir <- Directory.create(baseDir / "addons")
+          _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir)
+          _ <- Context
+            .userInputFile(baseDir)
+            .parent
+            .fold(().pure[F])(Files[F].createDirectories)
+          archives = addons.map(_.archive).distinctBy(_.url)
+          _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir")
+          mkCtxs <- archives.traverse { archive =>
+            for {
+              _ <- logger.debug(s"Extracting $archive")
+              addonDir <- archive.extractTo(urlReader, addonsDir)
+              meta <- AddonMeta.findInDirectory(addonDir)
+              mkCtx = (ref: AddonRef) =>
+                Context(ref, meta, baseDir, addonDir, outDir, cacheDir)
+            } yield archive.url -> mkCtx
+          }
+          ctxFactory = mkCtxs.toMap
+          res = addons.map(ref => ctxFactory(ref.archive.url)(ref))
+        } yield res
+
+      private def runAddon(logger: Logger[F], env: Env)(
+          ctx: Context
+      ): F[AddonResult] =
+        for {
+          _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}")
+          _ <- logger.trace("Storing user input into file")
+          _ <- Stream
+            .emit(ctx.addon.args)
+            .through(fs2.text.utf8.encode)
+            .through(Files[F].writeAll(ctx.userInputFile, Flags.Write))
+            .compile
+            .drain
+
+          runner <- selectRunner(cfg, ctx.meta, ctx.addonDir)
+          result <- runner.run(logger, env, ctx)
+        } yield result
+    }
+
+  def selectRunner[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      meta: AddonMeta,
+      addonDir: Path
+  ): F[AddonRunner[F]] =
+    for {
+      addonRunner <- meta.enabledTypes(Left(addonDir))
+      // intersect on list retains order in first
+      possibleRunner = cfg.runner
+        .intersect(addonRunner)
+        .map(AddonRunner.forType[F](cfg))
+      runner = possibleRunner match {
+        case Nil =>
+          AddonRunner.failWith(
+            s"No runner available for addon config ${meta.runner} and config ${cfg.runner}."
+          )
+        case list =>
+          AddonRunner.firstSuccessful(list)
+      }
+    } yield runner
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.addons.AddonExecutorConfig._
+import docspell.common.Duration
+import docspell.common.exec.{Args, SysCmd}
+
+case class AddonExecutorConfig(
+    runner: List[RunnerType],
+    runTimeout: Duration,
+    nspawn: NSpawn,
+    nixRunner: NixConfig,
+    dockerRunner: DockerConfig
+)
+
+object AddonExecutorConfig {
+
+  case class NSpawn(
+      enabled: Boolean,
+      sudoBinary: String,
+      nspawnBinary: String,
+      containerWait: Duration
+  ) {
+    val nspawnVersion =
+      SysCmd(nspawnBinary, Args.of("--version")).withTimeout(Duration.seconds(2))
+  }
+
+  case class NixConfig(
+      nixBinary: String,
+      buildTimeout: Duration
+  )
+
+  case class DockerConfig(
+      dockerBinary: String,
+      buildTimeout: Duration
+  ) {
+    def dockerBuild(imageName: String): SysCmd =
+      SysCmd(dockerBinary, "build", "-t", imageName, ".").withTimeout(buildTimeout)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.logging.Logger
+
+trait AddonLoggerExtension {
+
+  implicit final class LoggerAddonOps[F[_]](self: Logger[F]) {
+    private val addonName = "addon-name"
+    private val addonVersion = "addon-version"
+
+    def withAddon(r: AddonArchive): Logger[F] =
+      self.capture(addonName, r.name).capture(addonVersion, r.version)
+
+    def withAddon(r: Context): Logger[F] =
+      withAddon(r.addon.archive)
+
+    def withAddon(m: AddonMeta): Logger[F] =
+      self.capture(addonName, m.meta.name).capture(addonVersion, m.meta.version)
+  }
+}
+
+object AddonLoggerExtension extends AddonLoggerExtension

modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala

@@ -0,0 +1,216 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import java.io.FileNotFoundException
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common.Glob
+import docspell.files.Zip
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.yaml.{parser => YamlParser}
+import io.circe.{Decoder, Encoder}
+import io.circe.{parser => JsonParser}
+
+case class AddonMeta(
+    meta: AddonMeta.Meta,
+    triggers: Option[Set[AddonTriggerType]],
+    args: Option[List[String]],
+    runner: Option[AddonMeta.Runner],
+    options: Option[AddonMeta.Options]
+) {
+
+  def nameAndVersion: String =
+    s"${meta.name}-${meta.version}"
+
+  def parseResult: Boolean =
+    options.exists(_.collectOutput)
+
+  def ignoreResult: Boolean =
+    !parseResult
+
+  def isImpure: Boolean =
+    options.exists(_.isImpure)
+
+  def isPure: Boolean =
+    options.forall(_.isPure)
+
+  /** Returns a list of runner types that are possible to use for this addon. This is also
+    * inspecting the archive to return defaults when the addon isn't declaring it in the
+    * descriptor.
+    */
+  def enabledTypes[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[List[RunnerType]] =
+    for {
+      filesExists <- AddonArchive.dockerAndFlakeExists(archive)
+      (dockerFileExists, flakeFileExists) = filesExists
+
+      nixEnabled = runner.flatMap(_.nix).map(_.enable) match {
+        case Some(flag) => flag
+        case None       => flakeFileExists
+      }
+
+      dockerEnabled = runner.flatMap(_.docker).map(_.enable) match {
+        case Some(flag) => flag
+        case None       => dockerFileExists
+      }
+
+      trivialEnabled = runner.flatMap(_.trivial).exists(_.enable)
+
+      result = RunnerType.all.filter(_.fold(nixEnabled, dockerEnabled, trivialEnabled))
+    } yield result
+
+}
+
+object AddonMeta {
+
+  def empty(name: String, version: String): AddonMeta =
+    AddonMeta(Meta(name, version, None), None, None, None, None)
+
+  case class Meta(name: String, version: String, description: Option[String])
+  case class Runner(
+      nix: Option[NixRunner],
+      docker: Option[DockerRunner],
+      trivial: Option[TrivialRunner]
+  )
+  case class NixRunner(enable: Boolean)
+  case class DockerRunner(enable: Boolean, image: Option[String], build: Option[String])
+  case class TrivialRunner(enable: Boolean, exec: String)
+  case class Options(networking: Boolean, collectOutput: Boolean) {
+    def isPure = !networking && collectOutput
+    def isImpure = networking
+    def isUseless = !networking && !collectOutput
+    def isUseful = networking || collectOutput
+  }
+
+  object NixRunner {
+    implicit val jsonEncoder: Encoder[NixRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[NixRunner] =
+      deriveDecoder
+  }
+
+  object DockerRunner {
+    implicit val jsonEncoder: Encoder[DockerRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[DockerRunner] =
+      deriveDecoder
+  }
+
+  object TrivialRunner {
+    implicit val jsonEncoder: Encoder[TrivialRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[TrivialRunner] =
+      deriveDecoder
+  }
+
+  object Runner {
+    implicit val jsonEncoder: Encoder[Runner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Runner] =
+      deriveDecoder
+  }
+
+  object Options {
+    implicit val jsonEncoder: Encoder[Options] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Options] =
+      deriveDecoder
+  }
+
+  object Meta {
+    implicit val jsonEncoder: Encoder[Meta] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Meta] =
+      deriveDecoder
+  }
+
+  implicit val jsonEncoder: Encoder[AddonMeta] =
+    deriveEncoder
+
+  implicit val jsonDecoder: Decoder[AddonMeta] =
+    deriveDecoder
+
+  def fromJsonString(str: String): Either[Throwable, AddonMeta] =
+    JsonParser.decode[AddonMeta](str)
+
+  def fromJsonBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] =
+    bytes
+      .through(fs2.text.utf8.decode)
+      .compile
+      .string
+      .map(fromJsonString)
+      .rethrow
+
+  def fromYamlString(str: String): Either[Throwable, AddonMeta] =
+    YamlParser.parse(str).flatMap(_.as[AddonMeta])
+
+  def fromYamlBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] =
+    bytes
+      .through(fs2.text.utf8.decode)
+      .compile
+      .string
+      .map(fromYamlString)
+      .rethrow
+
+  def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = {
+    val logger = docspell.logging.getLogger[F]
+    val jsonFile = dir / "docspell-addon.json"
+    val yamlFile = dir / "docspell-addon.yaml"
+    val yamlFile2 = dir / "docspell-addon.yml"
+
+    OptionT
+      .liftF(Files[F].exists(jsonFile))
+      .flatTap(OptionT.whenF(_)(logger.debug(s"Reading json addon file $jsonFile")))
+      .flatMap(OptionT.whenF(_)(fromJsonBytes(Files[F].readAll(jsonFile))))
+      .orElse(
+        OptionT
+          .liftF(Files[F].exists(yamlFile))
+          .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile")))
+          .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile))))
+      )
+      .orElse(
+        OptionT
+          .liftF(Files[F].exists(yamlFile2))
+          .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile2")))
+          .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile2))))
+      )
+      .getOrElseF(
+        Sync[F].raiseError(
+          new FileNotFoundException(s"No docspell-addon.{yaml|json} file found in $dir!")
+        )
+      )
+  }
+
+  def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = {
+    val fail: F[AddonMeta] = Async[F].raiseError(
+      new FileNotFoundException(
+        s"No docspell-addon.{yaml|json} file found in zip!"
+      )
+    )
+    zipFile
+      .through(Zip.unzip(8192, Glob("**/docspell-addon.*")))
+      .filter(bin => !bin.name.endsWith("/"))
+      .flatMap { bin =>
+        if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data))
+        else if (bin.extensionIn(Set("yaml", "yml")))
+          Stream.eval(AddonMeta.fromYamlBytes(bin.data))
+        else Stream.empty
+      }
+      .take(1)
+      .compile
+      .last
+      .flatMap(_.map(Sync[F].pure).getOrElse(fail))
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+case class AddonRef(archive: AddonArchive, args: String)

modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+
+import docspell.addons.out.AddonOutput
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Codec, Decoder, Encoder}
+
+sealed trait AddonResult {
+  def toEither: Either[Throwable, AddonOutput]
+
+  def isSuccess: Boolean = toEither.isRight
+  def isFailure: Boolean = !isSuccess
+
+  def cast: AddonResult = this
+}
+
+object AddonResult {
+
+  /** The addon was run successful, but decoding its stdout failed. */
+  case class DecodingError(message: String) extends AddonResult {
+    def toEither = Left(new IllegalStateException(message))
+  }
+  object DecodingError {
+    implicit val jsonEncoder: Encoder[DecodingError] = deriveEncoder
+    implicit val jsonDecoder: Decoder[DecodingError] = deriveDecoder
+  }
+
+  def decodingError(message: String): AddonResult =
+    DecodingError(message)
+
+  def decodingError(ex: Throwable): AddonResult =
+    DecodingError(ex.getMessage)
+
+  /** Running the addon resulted in an invalid return code (!= 0). */
+  case class ExecutionError(rc: Int) extends AddonResult {
+    def toEither = Left(new IllegalStateException(s"Exit code: $rc"))
+  }
+
+  object ExecutionError {
+    implicit val jsonEncoder: Encoder[ExecutionError] = deriveEncoder
+    implicit val jsonDecoder: Decoder[ExecutionError] = deriveDecoder
+  }
+
+  def executionError(rc: Int): AddonResult =
+    ExecutionError(rc)
+
+  /** The execution of the addon failed with an exception. */
+  case class ExecutionFailed(error: Throwable) extends AddonResult {
+    def toEither = Left(error)
+  }
+
+  object ExecutionFailed {
+    implicit val throwableCodec: Codec[Throwable] =
+      Codec.from(
+        Decoder[String].emap(str => Right(ErrorMessageThrowable(str))),
+        Encoder[String].contramap(_.getMessage)
+      )
+
+    implicit val jsonEncoder: Encoder[ExecutionFailed] = deriveEncoder
+    implicit val jsonDecoder: Decoder[ExecutionFailed] = deriveDecoder
+
+    private class ErrorMessageThrowable(msg: String) extends RuntimeException(msg) {
+      override def fillInStackTrace() = this
+    }
+    private object ErrorMessageThrowable {
+      def apply(str: String): Throwable = new ErrorMessageThrowable(str)
+    }
+  }
+
+  def executionFailed(error: Throwable): AddonResult =
+    ExecutionFailed(error)
+
+  /** The addon was run successfully and its output was decoded (if any). */
+  case class Success(output: AddonOutput) extends AddonResult {
+    def toEither = Right(output)
+  }
+
+  object Success {
+    implicit val jsonEncoder: Encoder[Success] = deriveEncoder
+    implicit val jsonDecoder: Decoder[Success] = deriveDecoder
+  }
+
+  def success(output: AddonOutput): AddonResult =
+    Success(output)
+
+  val empty: AddonResult = Success(AddonOutput.empty)
+
+  def combine(a: AddonResult, b: AddonResult): AddonResult =
+    (a, b) match {
+      case (Success(o1), Success(o2)) => Success(AddonOutput.combine(o1, o2))
+      case (Success(_), e)            => e
+      case (e, Success(_))            => e
+      case _                          => a
+    }
+
+  implicit val deriveConfig: Configuration =
+    Configuration.default.withDiscriminator("result").withKebabCaseConstructorNames
+
+  implicit val jsonDecoder: Decoder[AddonResult] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AddonResult] = deriveConfiguredEncoder
+
+  implicit val addonResultMonoid: Monoid[AddonResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Applicative
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+
+import docspell.addons.runner._
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonRunner[F[_]] {
+  def runnerType: List[RunnerType]
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ): F[AddonResult]
+}
+
+object AddonRunner {
+  def forType[F[_]: Async](cfg: AddonExecutorConfig)(rt: RunnerType) =
+    rt match {
+      case RunnerType.NixFlake => NixFlakeRunner[F](cfg)
+      case RunnerType.Docker   => DockerRunner[F](cfg)
+      case RunnerType.Trivial  => TrivialRunner[F](cfg)
+    }
+
+  def failWith[F[_]](errorMsg: String)(implicit F: Applicative[F]): AddonRunner[F] =
+    pure(AddonResult.executionFailed(new Exception(errorMsg)))
+
+  def pure[F[_]: Applicative](result: AddonResult): AddonRunner[F] =
+    new AddonRunner[F] {
+      val runnerType = Nil
+
+      def run(logger: Logger[F], env: Env, ctx: Context) =
+        Applicative[F].pure(result)
+    }
+
+  def firstSuccessful[F[_]: Sync](runners: List[AddonRunner[F]]): AddonRunner[F] =
+    runners match {
+      case Nil      => failWith("No runner available!")
+      case a :: Nil => a
+      case _ =>
+        new AddonRunner[F] {
+          val runnerType = runners.flatMap(_.runnerType).distinct
+
+          def run(logger: Logger[F], env: Env, ctx: Context) =
+            Stream
+              .emits(runners)
+              .evalTap(r =>
+                logger.info(
+                  s"Attempt to run addon ${ctx.meta.nameAndVersion} with runner ${r.runnerType}"
+                )
+              )
+              .evalMap(_.run(logger, env, ctx))
+              .flatMap {
+                case r @ AddonResult.Success(_) => Stream.emit(r.cast.some)
+                case r @ AddonResult.ExecutionFailed(ex) =>
+                  if (ctx.meta.isPure) {
+                    logger.stream
+                      .warn(ex)(s"Addon runner failed, try next.")
+                      .as(r.cast.some)
+                  } else {
+                    logger.stream.warn(ex)(s"Addon runner failed!").as(None)
+                  }
+                case r @ AddonResult.ExecutionError(rc) =>
+                  if (ctx.meta.isPure) {
+                    logger.stream
+                      .warn(s"Addon runner returned non-zero: $rc. Try next.")
+                      .as(r.cast.some)
+                  } else {
+                    logger.stream.warn(s"Addon runner returned non-zero: $rc!").as(None)
+                  }
+                case AddonResult.DecodingError(message) =>
+                  // Don't retry as it is very unlikely that the output differs using another runner
+                  // This is most likely a bug in the addon
+                  logger.stream
+                    .warn(
+                      s"Error decoding the output of the addon ${ctx.meta.nameAndVersion}: $message. Stopping here. This is likely a bug in the addon."
+                    )
+                    .as(None)
+              }
+              .unNoneTerminate
+              .takeThrough(_.isFailure)
+              .compile
+              .last
+              .flatMap {
+                case Some(r) => r.pure[F]
+                case None =>
+                  AddonResult
+                    .executionFailed(new NoSuchElementException("No runner left :("))
+                    .pure[F]
+              }
+        }
+    }
+
+  def firstSuccessful[F[_]: Sync](
+      runner: AddonRunner[F],
+      runners: AddonRunner[F]*
+  ): AddonRunner[F] =
+    firstSuccessful(runner :: runners.toList)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.NonEmptyList
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait AddonTriggerType {
+  def name: String
+}
+
+object AddonTriggerType {
+
+  /** The final step when processing an item. */
+  case object FinalProcessItem extends AddonTriggerType {
+    val name = "final-process-item"
+  }
+
+  /** The final step when reprocessing an item. */
+  case object FinalReprocessItem extends AddonTriggerType {
+    val name = "final-reprocess-item"
+  }
+
+  /** Running periodically based on a schedule. */
+  case object Scheduled extends AddonTriggerType {
+    val name = "scheduled"
+  }
+
+  /** Running (manually) on some existing item. */
+  case object ExistingItem extends AddonTriggerType {
+    val name = "existing-item"
+  }
+
+  val all: NonEmptyList[AddonTriggerType] =
+    NonEmptyList.of(FinalProcessItem, FinalReprocessItem, Scheduled, ExistingItem)
+
+  def fromString(str: String): Either[String, AddonTriggerType] =
+    all
+      .find(e => e.name.equalsIgnoreCase(str))
+      .toRight(s"Invalid addon trigger type: $str")
+
+  def unsafeFromString(str: String): AddonTriggerType =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonEncoder: Encoder[AddonTriggerType] =
+    Encoder.encodeString.contramap(_.name)
+
+  implicit val jsonDecoder: Decoder[AddonTriggerType] =
+    Decoder.decodeString.emap(fromString)
+}

modules/addonlib/src/main/scala/docspell/addons/Context.scala

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import fs2.io.file.Path
+
+import docspell.common._
+import docspell.common.exec.{Args, Env, SysCmd}
+
+/** Context a list of addons is executed in.
+  *
+  * Each addon has its own `addonDir`, but all share the same `baseDir` in one run.
+  */
+case class Context(
+    addon: AddonRef,
+    meta: AddonMeta,
+    baseDir: Path,
+    addonDir: Path,
+    outputDir: Path,
+    cacheDir: Path
+) {
+  def userInputFile = Context.userInputFile(baseDir)
+  def tempDir = Context.tempDir(baseDir)
+
+  private[addons] def addonCommand(
+      binary: String,
+      timeout: Duration,
+      relativeToBase: Boolean,
+      outputDir: Option[String],
+      cacheDir: Option[String]
+  ): SysCmd = {
+    val execBin = Option
+      .when(relativeToBase)(binary)
+      .getOrElse((baseDir / binary).toString)
+
+    val input = Option
+      .when(relativeToBase)(baseDir.relativize(userInputFile))
+      .getOrElse(userInputFile)
+
+    val allArgs =
+      Args(meta.args.getOrElse(Nil)).append(input)
+    val envAddonDir = Option
+      .when(relativeToBase)(baseDir.relativize(addonDir))
+      .getOrElse(addonDir)
+    val envTmpDir = Option
+      .when(relativeToBase)(baseDir.relativize(tempDir))
+      .getOrElse(tempDir)
+    val outDir = outputDir.getOrElse(this.outputDir.toString)
+    val cache = cacheDir.getOrElse(this.cacheDir.toString)
+    val moreEnv =
+      Env.of(
+        "ADDON_DIR" -> envAddonDir.toString,
+        "TMPDIR" -> envTmpDir.toString,
+        "TMP_DIR" -> envTmpDir.toString,
+        "OUTPUT_DIR" -> outDir,
+        "CACHE_DIR" -> cache
+      )
+
+    SysCmd(execBin, allArgs).withTimeout(timeout).addEnv(moreEnv)
+  }
+}
+
+object Context {
+  def userInputFile(base: Path): Path =
+    base / "arguments" / "user-input"
+  def tempDir(base: Path): Path =
+    base / "temp"
+}

modules/addonlib/src/main/scala/docspell/addons/Directory.scala

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import cats.{Applicative, Monad}
+import fs2.io.file.{Files, Path, PosixPermissions}
+
+object Directory {
+
+  def create[F[_]: Files: Applicative](dir: Path): F[Path] =
+    Files[F]
+      .createDirectories(dir, PosixPermissions.fromOctal("777"))
+      .as(dir)
+
+  def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] =
+    (dir :: dirs.toList).traverse_(Files[F].createDirectories(_))
+
+  def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
+    List(
+      Files[F].isDirectory(dir),
+      Files[F].list(dir).take(1).compile.last.map(_.isDefined)
+    ).sequence.map(_.forall(identity))
+
+  def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
+    nonEmpty(dir).map(b => !b)
+
+  def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
+    for {
+      _ <- Resource.eval(Files[F].createDirectories(parent))
+      d <- mkTemp(parent, prefix)
+    } yield d
+
+  def temp2[F[_]: Files](
+      parent: Path,
+      prefix1: String,
+      prefix2: String
+  ): Resource[F, (Path, Path)] =
+    for {
+      _ <- Resource.eval(Files[F].createDirectories(parent))
+      a <- mkTemp(parent, prefix1)
+      b <- mkTemp(parent, prefix2)
+    } yield (a, b)
+
+  def createTemp[F[_]: Files: Monad](
+      parent: Path,
+      prefix: String
+  ): F[Path] =
+    for {
+      _ <- Files[F].createDirectories(parent)
+      d <- mkTemp_(parent, prefix)
+    } yield d
+
+  private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
+    Files[F]
+      .tempDirectory(
+        parent.some,
+        prefix,
+        PosixPermissions.fromOctal("777")
+      )
+
+  private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] =
+    Files[F]
+      .createTempDirectory(
+        parent.some,
+        prefix,
+        PosixPermissions.fromOctal("777")
+      )
+}

modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect.Resource
+import fs2.io.file.{Files, Path}
+
+import docspell.common.exec.Env
+
+case class InputEnv(
+    addons: List[AddonRef],
+    baseDir: Path,
+    outputDir: Path,
+    cacheDir: Path,
+    env: Env
+) {
+  def addEnv(key: String, value: String): InputEnv =
+    copy(env = env.add(key, value))
+
+  def addEnv(vp: (String, String)*): InputEnv =
+    copy(env = env.addAll(vp.toMap))
+
+  def addEnv(vm: Map[String, String]): InputEnv =
+    copy(env = env ++ Env(vm))
+
+  def withTempBase[F[_]: Files]: Resource[F, InputEnv] =
+    Directory.temp(baseDir, "addon-").map(path => copy(baseDir = path))
+}

modules/addonlib/src/main/scala/docspell/addons/Middleware.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monad
+import cats.data.Kleisli
+import cats.effect.kernel.Sync
+import cats.syntax.all._
+import fs2.io.file.Files
+
+trait Middleware[F[_]] extends (AddonExec[F] => AddonExec[F]) { self =>
+
+  def >>(next: Middleware[F]): Middleware[F] =
+    Middleware(self.andThen(next))
+}
+
+object Middleware {
+  def apply[F[_]](f: AddonExec[F] => AddonExec[F]): Middleware[F] =
+    a => f(a)
+
+  def identity[F[_]]: Middleware[F] = Middleware(scala.Predef.identity)
+
+  /** Uses a temporary base dir that is removed after execution. Use this as the last
+    * layer!
+    */
+  def ephemeralRun[F[_]: Files: Sync]: Middleware[F] =
+    Middleware(a => Kleisli(_.withTempBase.use(a.run)))
+
+  /** Prepare running an addon */
+  def prepare[F[_]: Monad](
+      prep: Kleisli[F, InputEnv, InputEnv]
+  ): Middleware[F] =
+    Middleware(a => Kleisli(in => prep.run(in).flatMap(a.run)))
+
+  def postProcess[F[_]: Monad](
+      post: Kleisli[F, AddonExecutionResult, Unit]
+  ): Middleware[F] =
+    Middleware(_.flatMapF(r => post.map(_ => r).run(r)))
+}

modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.NonEmptyList
+import cats.syntax.all._
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait RunnerType {
+  def name: String
+
+  def fold[A](
+      nixFlake: => A,
+      docker: => A,
+      trivial: => A
+  ): A
+}
+object RunnerType {
+  case object NixFlake extends RunnerType {
+    val name = "nix-flake"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = nixFlake
+  }
+  case object Docker extends RunnerType {
+    val name = "docker"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = docker
+  }
+  case object Trivial extends RunnerType {
+    val name = "trivial"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = trivial
+  }
+
+  val all: NonEmptyList[RunnerType] =
+    NonEmptyList.of(NixFlake, Docker, Trivial)
+
+  def fromString(str: String): Either[String, RunnerType] =
+    all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid runner value: $str")
+
+  def unsafeFromString(str: String): RunnerType =
+    fromString(str).fold(sys.error, identity)
+
+  def fromSeparatedString(str: String): Either[String, List[RunnerType]] =
+    str.split("[\\s,]+").toList.map(_.trim).traverse(fromString)
+
+  implicit val jsonDecoder: Decoder[RunnerType] =
+    Decoder[String].emap(RunnerType.fromString)
+
+  implicit val jsonEncoder: Encoder[RunnerType] =
+    Encoder[String].contramap(_.name)
+}

modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.kernel.Monoid
+
+import docspell.common.bc.BackendCommand
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Decoded stdout result from executing an addon. */
+case class AddonOutput(
+    commands: List[BackendCommand] = Nil,
+    files: List[ItemFile] = Nil,
+    newItems: List[NewItem] = Nil
+)
+
+object AddonOutput {
+  val empty: AddonOutput = AddonOutput()
+
+  def combine(a: AddonOutput, b: AddonOutput): AddonOutput =
+    AddonOutput(a.commands ++ b.commands, a.files ++ b.files)
+
+  implicit val addonResultMonoid: Monoid[AddonOutput] =
+    Monoid.instance(empty, combine)
+
+  implicit val jsonConfig: Configuration =
+    Configuration.default.withDefaults
+
+  implicit val jsonDecoder: Decoder[AddonOutput] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AddonOutput] = deriveConfiguredEncoder
+
+  def fromString(str: String): Either[Throwable, AddonOutput] =
+    io.circe.parser.decode[AddonOutput](str)
+
+  def unsafeFromString(str: String): AddonOutput =
+    fromString(str).fold(throw _, identity)
+}

modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.FileSupport._
+import docspell.logging.Logger
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Addons can produce files in their output directory. These can be named here in order
+  * to do something with them.
+  *
+  *   - textFiles will replace the extracted text with the contents of the file
+  *   - pdfFiles will add/replace the converted pdf with the given file
+  *   - previewImages will add/replace preview images
+  *   - newFiles will be added as new attachments to the item
+  *
+  * Files must be referenced by attachment id.
+  */
+final case class ItemFile(
+    itemId: Ident,
+    textFiles: Map[String, String] = Map.empty,
+    pdfFiles: Map[String, String] = Map.empty,
+    previewImages: Map[String, String] = Map.empty,
+    newFiles: List[NewFile] = Nil
+) {
+  def isEmpty: Boolean =
+    textFiles.isEmpty && pdfFiles.isEmpty && previewImages.isEmpty
+
+  def nonEmpty: Boolean = !isEmpty
+
+  def resolveTextFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.text("*"), textFiles)
+
+  def resolvePdfFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.pdf, pdfFiles)
+
+  def resolvePreviewFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.image("*"), previewImages)
+
+  def resolveNewFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(NewFile, Path)]] =
+    newFiles.traverseFilter(nf =>
+      nf.resolveFile(logger, outputDir).map(_.map(p => (nf, p)))
+    )
+
+  private def resolveFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path,
+      mime: MimeType,
+      files: Map[String, String]
+  ): F[List[(String, Path)]] = {
+    val allFiles =
+      files.toList.map(t => t._1 -> outputDir / t._2)
+
+    allFiles.traverseFilter { case (key, file) =>
+      OptionT(file.detectMime)
+        .flatMapF(fileType =>
+          if (mime.matches(fileType)) (key -> file).some.pure[F]
+          else
+            logger
+              .warn(
+                s"File $file provided as ${mime.asString} file, but was recognized as ${fileType.asString}. Ignoring it."
+              )
+              .as(None: Option[(String, Path)])
+        )
+        .value
+    }
+  }
+}
+
+object ItemFile {
+
+  implicit val jsonConfig: Configuration =
+    Configuration.default.withDefaults
+
+  implicit val jsonEncoder: Encoder[ItemFile] = deriveConfiguredEncoder
+  implicit val jsonDecoder: Decoder[ItemFile] = deriveConfiguredDecoder
+}

modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.effect.Sync
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.out.NewFile.Meta
+import docspell.common.ProcessItemArgs.ProcessMeta
+import docspell.common.{Ident, Language}
+import docspell.logging.Logger
+
+import io.circe.Codec
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.deriveConfiguredCodec
+import io.circe.generic.semiauto.deriveCodec
+
+case class NewFile(metadata: Meta = Meta.empty, file: String) {
+
+  def resolveFile[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[Option[Path]] = {
+    val target = outputDir / file
+    Files[F]
+      .exists(target)
+      .flatMap(flag =>
+        if (flag) target.some.pure[F]
+        else logger.warn(s"File not found: $file").as(Option.empty)
+      )
+  }
+}
+
+object NewFile {
+
+  case class Meta(
+      language: Option[Language],
+      skipDuplicate: Option[Boolean],
+      attachmentsOnly: Option[Boolean]
+  ) {
+
+    def toProcessMeta(
+        cid: Ident,
+        itemId: Ident,
+        collLang: Option[Language],
+        sourceAbbrev: String
+    ): ProcessMeta =
+      ProcessMeta(
+        collective = cid,
+        itemId = Some(itemId),
+        language = language.orElse(collLang).getOrElse(Language.English),
+        direction = None,
+        sourceAbbrev = sourceAbbrev,
+        folderId = None,
+        validFileTypes = Seq.empty,
+        skipDuplicate = skipDuplicate.getOrElse(true),
+        fileFilter = None,
+        tags = None,
+        reprocess = false,
+        attachmentsOnly = attachmentsOnly
+      )
+  }
+
+  object Meta {
+    val empty = Meta(None, None, None)
+    implicit val jsonCodec: Codec[Meta] = deriveCodec
+  }
+
+  implicit val jsonConfig: Configuration = Configuration.default.withDefaults
+
+  implicit val jsonCodec: Codec[NewFile] = deriveConfiguredCodec
+}

modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.Monad
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.out.NewItem.Meta
+import docspell.common._
+import docspell.logging.Logger
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+case class NewItem(metadata: Option[Meta], files: List[String]) {
+
+  def toProcessMeta(
+      cid: Ident,
+      collLang: Option[Language],
+      sourceAbbrev: String
+  ): ProcessItemArgs.ProcessMeta =
+    metadata
+      .getOrElse(Meta(None, None, None, None, None, None, None))
+      .toProcessArgs(cid, collLang, sourceAbbrev)
+
+  def resolveFiles[F[_]: Files: Monad](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[Path]] = {
+    val allFiles =
+      files.map(name => outputDir / name)
+
+    allFiles.traverseFilter { file =>
+      Files[F]
+        .exists(file)
+        .flatMap {
+          case true => file.some.pure[F]
+          case false =>
+            logger
+              .warn(s"File $file doesn't exist. Ignoring it.")
+              .as(None)
+        }
+    }
+  }
+}
+
+object NewItem {
+
+  case class Meta(
+      language: Option[Language],
+      direction: Option[Direction],
+      folderId: Option[Ident],
+      source: Option[String],
+      skipDuplicate: Option[Boolean],
+      tags: Option[List[String]],
+      attachmentsOnly: Option[Boolean]
+  ) {
+
+    def toProcessArgs(
+        cid: Ident,
+        collLang: Option[Language],
+        sourceAbbrev: String
+    ): ProcessItemArgs.ProcessMeta =
+      ProcessItemArgs.ProcessMeta(
+        collective = cid,
+        itemId = None,
+        language = language.orElse(collLang).getOrElse(Language.English),
+        direction = direction,
+        sourceAbbrev = source.getOrElse(sourceAbbrev),
+        folderId = folderId,
+        validFileTypes = Seq.empty,
+        skipDuplicate = skipDuplicate.getOrElse(true),
+        fileFilter = None,
+        tags = tags,
+        reprocess = false,
+        attachmentsOnly = attachmentsOnly
+      )
+  }
+
+  object Meta {
+    implicit val jsonEncoder: Encoder[Meta] = deriveEncoder
+    implicit val jsonDecoder: Decoder[Meta] = deriveDecoder
+  }
+
+  implicit val jsonDecoder: Decoder[NewItem] = deriveDecoder
+  implicit val jsonEncoder: Encoder[NewItem] = deriveEncoder
+}

modules/addonlib/src/main/scala/docspell/addons/package.scala

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell
+
+import cats.data.Kleisli
+
+package object addons {
+
+  type AddonExec[F[_]] = Kleisli[F, InputEnv, AddonExecutionResult]
+
+}

modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.Applicative
+import cats.effect.{Ref, Sync}
+import cats.syntax.all._
+import fs2.Pipe
+
+trait CollectOut[F[_]] {
+
+  def get: F[String]
+
+  def append: Pipe[F, String, String]
+}
+
+object CollectOut {
+
+  def none[F[_]: Applicative]: CollectOut[F] =
+    new CollectOut[F] {
+      def get = "".pure[F]
+      def append = identity
+    }
+
+  def buffer[F[_]: Sync]: F[CollectOut[F]] =
+    Ref
+      .of[F, Vector[String]](Vector.empty)
+      .map(buffer =>
+        new CollectOut[F] {
+          override def get =
+            buffer.get.map(_.mkString("\n").trim)
+
+          override def append =
+            _.evalTap(line =>
+              if (line.trim.nonEmpty) buffer.update(_.appended(line)) else ().pure[F]
+            )
+        }
+      )
+}

modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import fs2.io.file.Path
+
+import docspell.common.Duration
+import docspell.common.exec.{Args, Env, SysCmd}
+
+/** Builder for a docker system command. */
+case class DockerBuilder(
+    dockerBinary: String,
+    subCmd: String,
+    timeout: Duration,
+    containerName: Option[String] = None,
+    env: Env = Env.empty,
+    mounts: Args = Args.empty,
+    network: Option[String] = Some("host"),
+    workingDir: Option[String] = None,
+    imageName: Option[String] = None,
+    cntCmd: Args = Args.empty
+) {
+  def containerCmd(args: Args): DockerBuilder =
+    copy(cntCmd = args)
+  def containerCmd(args: Seq[String]): DockerBuilder =
+    copy(cntCmd = Args(args))
+
+  def imageName(name: String): DockerBuilder =
+    copy(imageName = Some(name))
+
+  def workDirectory(dir: String): DockerBuilder =
+    copy(workingDir = Some(dir))
+
+  def withDockerBinary(bin: String): DockerBuilder =
+    copy(dockerBinary = bin)
+
+  def withSubCmd(cmd: String): DockerBuilder =
+    copy(subCmd = cmd)
+
+  def withEnv(key: String, value: String): DockerBuilder =
+    copy(env = env.add(key, value))
+
+  def withEnv(moreEnv: Env): DockerBuilder =
+    copy(env = env ++ moreEnv)
+
+  def privateNetwork(flag: Boolean): DockerBuilder =
+    if (flag) copy(network = Some("none"))
+    else copy(network = Some("host"))
+
+  def mount(
+      hostDir: Path,
+      cntDir: Option[String] = None,
+      readOnly: Boolean = true
+  ): DockerBuilder = {
+    val target = cntDir.getOrElse(hostDir.toString)
+    val ro = Option.when(readOnly)(",readonly").getOrElse("")
+    val opt = s"type=bind,source=$hostDir,target=$target${ro}"
+    copy(mounts = mounts.append("--mount", opt))
+  }
+
+  def withName(containerName: String): DockerBuilder =
+    copy(containerName = Some(containerName))
+
+  def build: SysCmd =
+    SysCmd(dockerBinary, buildArgs).withTimeout(timeout)
+
+  private def buildArgs: Args =
+    Args
+      .of(subCmd)
+      .append("--rm")
+      .option("--name", containerName)
+      .append(mounts)
+      .option("--network", network)
+      .append(env.mapConcat((k, v) => List("--env", s"${k}=${v}")))
+      .option("-w", workingDir)
+      .appendOpt(imageName)
+      .append(cntCmd)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.addons.AddonExecutorConfig.DockerConfig
+import docspell.addons._
+import docspell.common.Duration
+import docspell.common.exec.{Env, SysCmd, SysExec}
+import docspell.common.util.Random
+import docspell.logging.Logger
+
+final class DockerRunner[F[_]: Async](cfg: DockerRunner.Config) extends AddonRunner[F] {
+
+  val runnerType = List(RunnerType.Docker)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ) = for {
+    _ <- OptionT.whenF(requireBuild(ctx))(build(logger, ctx)).value
+    suffix <- Random[F].string(4)
+    cmd = createDockerCommand(env, ctx, suffix)
+    result <- RunnerUtil.runAddonCommand(logger, cmd, ctx)
+  } yield result
+
+  def createDockerCommand(
+      env: Env,
+      ctx: Context,
+      suffix: String
+  ): SysCmd = {
+    val outputPath = "/mnt/output"
+    val cachePath = "/mnt/cache"
+    val addonArgs =
+      ctx.addonCommand(
+        "",
+        Duration.zero,
+        relativeToBase = true,
+        outputPath.some,
+        cachePath.some
+      )
+
+    DockerBuilder(cfg.docker.dockerBinary, "run", cfg.timeout)
+      .withName(ctx.meta.nameAndVersion + "-" + suffix)
+      .withEnv(env)
+      .withEnv(addonArgs.env)
+      .mount(ctx.baseDir, "/mnt/work".some, readOnly = false)
+      .mount(ctx.outputDir, outputPath.some, readOnly = false)
+      .mount(ctx.cacheDir, cachePath.some, readOnly = false)
+      .workDirectory("/mnt/work")
+      .privateNetwork(ctx.meta.isPure)
+      .imageName(imageName(ctx))
+      .containerCmd(addonArgs.args)
+      .build
+  }
+
+  def build(logger: Logger[F], ctx: Context): F[Unit] =
+    for {
+      _ <- logger.info(s"Building docker image for addon ${ctx.meta.nameAndVersion}")
+      cmd = cfg.docker.dockerBuild(imageName(ctx))
+      _ <- SysExec(cmd, logger, ctx.addonDir.some)
+        .flatMap(_.logOutputs(logger, "docker build"))
+        .use(_.waitFor())
+      _ <- logger.info(s"Docker image built successfully")
+    } yield ()
+
+  private def requireBuild(ctx: Context) =
+    ctx.meta.runner
+      .flatMap(_.docker)
+      .flatMap(_.image)
+      .isEmpty
+
+  private def imageName(ctx: Context): String =
+    ctx.meta.runner
+      .flatMap(_.docker)
+      .flatMap(_.image)
+      .getOrElse(s"${ctx.meta.meta.name}:latest")
+}
+
+object DockerRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): DockerRunner[F] =
+    new DockerRunner[F](Config(cfg.dockerRunner, cfg.runTimeout))
+
+  case class Config(docker: DockerConfig, timeout: Duration)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import fs2.io.file.Path
+
+import docspell.common.exec.{Args, Env, SysCmd}
+
+case class NSpawnBuilder(
+    child: SysCmd,
+    chroot: Path,
+    spawnBinary: String = "systemd-nspawn",
+    sudoBinary: String = "sudo",
+    args: Args = Args.empty,
+    env: Env = Env.empty
+) {
+
+  def withNSpawnBinary(bin: String): NSpawnBuilder =
+    copy(spawnBinary = bin)
+
+  def withSudoBinary(bin: String): NSpawnBuilder =
+    copy(sudoBinary = bin)
+
+  def withEnv(key: String, value: String): NSpawnBuilder =
+    copy(args = args.append(s"--setenv=$key=$value"))
+
+  def withEnvOpt(key: String, value: Option[String]): NSpawnBuilder =
+    value.map(v => withEnv(key, v)).getOrElse(this)
+
+  def withName(containerName: String): NSpawnBuilder =
+    copy(args = args.append(s"--machine=$containerName"))
+
+  def mount(
+      hostDir: Path,
+      cntDir: Option[String] = None,
+      readOnly: Boolean = true
+  ): NSpawnBuilder = {
+    val bind = if (readOnly) "--bind-ro" else "--bind"
+    val target = cntDir.map(dir => s":$dir").getOrElse("")
+    copy(args = args.append(s"${bind}=${hostDir}${target}"))
+  }
+
+  def workDirectory(dir: String): NSpawnBuilder =
+    copy(args = args.append(s"--chdir=$dir"))
+
+  def portMap(port: Int): NSpawnBuilder =
+    copy(args = args.append("-p", port.toString))
+
+  def privateNetwork(flag: Boolean): NSpawnBuilder =
+    if (flag) copy(args = args.append("--private-network"))
+    else this
+
+  def build: SysCmd =
+    SysCmd(
+      program = if (sudoBinary.nonEmpty) sudoBinary else spawnBinary,
+      args = buildArgs,
+      timeout = child.timeout,
+      env = env
+    )
+
+  private def buildArgs: Args =
+    Args
+      .of("--private-users=identity") // can't use -U because need writeable bind mounts
+      .append("--notify-ready=yes")
+      .append("--ephemeral")
+      .append("--as-pid2")
+      .append("--console=pipe")
+      .append("--no-pager")
+      .append("--bind-ro=/bin")
+      .append("--bind-ro=/usr/bin")
+      .append("--bind-ro=/nix/store")
+      .append(s"--directory=$chroot")
+      .append(args)
+      .append(child.env.map((n, v) => s"--setenv=$n=$v"))
+      .prependWhen(sudoBinary.nonEmpty)(spawnBinary)
+      .append("--")
+      .append(child.program)
+      .append(child.args)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.AddonExecutorConfig.{NSpawn, NixConfig}
+import docspell.addons._
+import docspell.addons.runner.NixFlakeRunner.PreCtx
+import docspell.common.Duration
+import docspell.common.exec._
+import docspell.logging.Logger
+
+final class NixFlakeRunner[F[_]: Async](cfg: NixFlakeRunner.Config)
+    extends AddonRunner[F] {
+
+  val runnerType = List(RunnerType.NixFlake)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ): F[AddonResult] =
+    prepare(logger, ctx)
+      .flatMap { preCtx =>
+        if (preCtx.nspawnEnabled) runInContainer(logger, env, preCtx, ctx)
+        else runOnHost(logger, env, preCtx, ctx)
+      }
+
+  def prepare(logger: Logger[F], ctx: Context): F[PreCtx] =
+    for {
+      _ <- logger.info(s"Prepare addon ${ctx.meta.nameAndVersion} for executing via nix")
+      _ <- logger.debug(s"Building with nix build")
+      _ <- SysExec(cfg.nixBuild, logger, workdir = ctx.addonDir.some)
+        .flatMap(_.logOutputs(logger, "nix build"))
+        .use(_.waitFor())
+      bin <- findFile(ctx.addonDir / "result" / "bin", ctx.addonDir / "result")
+      _ <- logger.debug(s"Build done, found binary: $bin")
+      _ <- logger.debug(s"Checking for systemd-nspawn…")
+      cnt <- checkContainer(logger)
+      _ <-
+        if (cnt)
+          logger.debug(s"Using systemd-nspawn to run addon in a container.")
+        else
+          logger.info(s"Running via systemd-nspawn is disabled in the config file")
+    } yield PreCtx(cnt, ctx.baseDir.relativize(bin))
+
+  private def checkContainer(logger: Logger[F]): F[Boolean] =
+    if (!cfg.nspawn.enabled) false.pure[F]
+    else RunnerUtil.checkContainer(logger, cfg.nspawn)
+
+  private def runOnHost(
+      logger: Logger[F],
+      env: Env,
+      preCtx: PreCtx,
+      ctx: Context
+  ): F[AddonResult] = {
+    val cmd =
+      SysCmd(preCtx.binary.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env)
+    RunnerUtil.runDirectly(logger, ctx)(cmd)
+  }
+
+  private def runInContainer(
+      logger: Logger[F],
+      env: Env,
+      preCtx: PreCtx,
+      ctx: Context
+  ): F[AddonResult] = {
+    val cmd = SysCmd(preCtx.binary.toString, Args.empty)
+      .withTimeout(cfg.timeout)
+      .addEnv(env)
+    RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd)
+  }
+
+  /** Find first file, try directories in given order. */
+  private def findFile(firstDir: Path, more: Path*): F[Path] = {
+    val fail: F[Path] = Sync[F].raiseError(
+      new NoSuchElementException(
+        s"No file found to execute in ${firstDir :: more.toList}"
+      )
+    )
+
+    Stream
+      .emits(more)
+      .cons1(firstDir)
+      .flatMap(dir =>
+        Files[F]
+          .list(dir)
+          .evalFilter(p => Files[F].isDirectory(p).map(!_))
+          .take(1)
+      )
+      .take(1)
+      .compile
+      .last
+      .flatMap(_.fold(fail)(Sync[F].pure))
+  }
+}
+
+object NixFlakeRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): NixFlakeRunner[F] =
+    new NixFlakeRunner[F](Config(cfg.nixRunner, cfg.nspawn, cfg.runTimeout))
+
+  case class Config(
+      nix: NixConfig,
+      nspawn: NSpawn,
+      timeout: Duration
+  ) {
+
+    val nixBuild =
+      SysCmd(nix.nixBinary, Args.of("build")).withTimeout(nix.buildTimeout)
+
+    val nspawnVersion = nspawn.nspawnVersion
+  }
+
+  case class PreCtx(nspawnEnabled: Boolean, binary: Path)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect.{Async, Sync}
+import cats.syntax.all._
+import fs2.Pipe
+import fs2.io.file.Files
+
+import docspell.addons._
+import docspell.addons.out.AddonOutput
+import docspell.common.exec.{SysCmd, SysExec}
+import docspell.common.util.Random
+import docspell.logging.Logger
+
+import io.circe.{parser => JsonParser}
+
+private[addons] object RunnerUtil {
+
+  /** Run the given `cmd` on this machine.
+    *
+    * The `cmd` is containing a template command to execute the addon. The path are
+    * expected to be relative to the `ctx.baseDir`. Additional arguments and environment
+    * variables are added as configured in the addon.
+    */
+  def runDirectly[F[_]: Async](
+      logger: Logger[F],
+      ctx: Context
+  )(cmd: SysCmd): F[AddonResult] = {
+    val addonCmd = ctx
+      .addonCommand(cmd.program, cmd.timeout, relativeToBase = false, None, None)
+      .withArgs(_.append(cmd.args))
+      .addEnv(cmd.env)
+    runAddonCommand(logger, addonCmd, ctx)
+  }
+
+  /** Run the given `cmd` inside a container via systemd-nspawn.
+    *
+    * The `cmd` is containing a template command to execute the addon. The path are
+    * expected to be relative to the `ctx.baseDir`. Additional arguments and environment
+    * variables are added as configured in the addon.
+    */
+  def runInContainer[F[_]: Async](
+      logger: Logger[F],
+      cfg: AddonExecutorConfig.NSpawn,
+      ctx: Context
+  )(cmd: SysCmd): F[AddonResult] = {
+    val outputPath = "/mnt/output"
+    val cachePath = "/mnt/cache"
+    val addonCmd = ctx
+      .addonCommand(
+        cmd.program,
+        cmd.timeout,
+        relativeToBase = true,
+        outputPath.some,
+        cachePath.some
+      )
+      .withArgs(_.append(cmd.args))
+      .addEnv(cmd.env)
+
+    val chroot = ctx.baseDir / "cnt-root"
+    val nspawn = NSpawnBuilder(addonCmd, chroot)
+      .withNSpawnBinary(cfg.nspawnBinary)
+      .withSudoBinary(cfg.sudoBinary)
+      .mount(ctx.baseDir, "/mnt/work".some, readOnly = false)
+      .mount(ctx.cacheDir, cachePath.some, readOnly = false)
+      .mount(ctx.outputDir, outputPath.some, readOnly = false)
+      .workDirectory("/mnt/work")
+      .withEnv("XDG_RUNTIME_DIR", "/mnt/work")
+      .privateNetwork(ctx.meta.isPure)
+
+    for {
+      suffix <- Random[F].string(4)
+      _ <- List(chroot).traverse_(Files[F].createDirectories)
+      res <- runAddonCommand(
+        logger,
+        nspawn.withName(ctx.meta.nameAndVersion + "-" + suffix).build,
+        ctx
+      )
+      // allow some time to unregister the current container
+      // only important when same addons are called in sequence too fast
+      _ <- Sync[F].sleep(cfg.containerWait.toScala)
+    } yield res
+  }
+
+  private def procPipe[F[_]](
+      p: String,
+      ctx: Context,
+      collect: CollectOut[F],
+      logger: Logger[F]
+  ): Pipe[F, String, Unit] =
+    _.through(collect.append)
+      .map(line => s">> [${ctx.meta.nameAndVersion} ($p)] $line")
+      .evalMap(logger.debug(_))
+
+  /** Runs the external command that is executing the addon.
+    *
+    * If the addons specifies to collect its output, the stdout is parsed as json and
+    * decoded into [[AddonOutput]].
+    */
+  def runAddonCommand[F[_]: Async](
+      logger: Logger[F],
+      cmd: SysCmd,
+      ctx: Context
+  ): F[AddonResult] =
+    for {
+      stdout <-
+        if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F]
+        else CollectOut.none[F].pure[F]
+      cmdResult <- SysExec(cmd, logger, ctx.baseDir.some)
+        .flatMap(
+          _.consumeOutputs(
+            procPipe("out", ctx, stdout, logger),
+            procPipe("err", ctx, CollectOut.none[F], logger)
+          )
+        )
+        .use(_.waitFor())
+        .attempt
+      addonResult <- cmdResult match {
+        case Right(rc) if rc != 0 =>
+          for {
+            _ <- logger.error(
+              s"Addon ${ctx.meta.nameAndVersion} returned non-zero: $rc"
+            )
+          } yield AddonResult.executionError(rc)
+
+        case Right(_) =>
+          for {
+            _ <- logger.debug(s"Addon ${ctx.meta.nameAndVersion} executed successfully!")
+            out <- stdout.get
+            _ <- logger.debug(s"Addon stdout: $out")
+            result = Option
+              .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)(
+                JsonParser
+                  .decode[AddonOutput](out)
+                  .fold(AddonResult.decodingError, AddonResult.success)
+              )
+              .getOrElse(AddonResult.empty)
+          } yield result
+
+        case Left(ex) =>
+          logger
+            .error(ex)(s"Executing external command failed!")
+            .as(AddonResult.executionFailed(ex))
+      }
+    } yield addonResult
+
+  /** Check whether `systemd-nspawn` is available on this machine. */
+  def checkContainer[F[_]: Async](
+      logger: Logger[F],
+      cfg: AddonExecutorConfig.NSpawn
+  ): F[Boolean] =
+    for {
+      rc <- SysExec(cfg.nspawnVersion, logger)
+        .flatMap(_.logOutputs(logger, "nspawn"))
+        .use(_.waitFor())
+      _ <-
+        OptionT
+          .whenF(rc != 0)(
+            logger.warn(
+              s"No systemd-nspawn found! Addon is not executed inside a container."
+            )
+          )
+          .value
+    } yield rc == 0
+}

modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect._
+import cats.kernel.Monoid
+import cats.syntax.all._
+import fs2.io.file.PosixPermission._
+import fs2.io.file.{Files, PosixPermissions}
+
+import docspell.addons.AddonExecutorConfig.NSpawn
+import docspell.addons._
+import docspell.common.Duration
+import docspell.common.exec.{Args, Env, SysCmd}
+import docspell.logging.Logger
+
+final class TrivialRunner[F[_]: Async](cfg: TrivialRunner.Config) extends AddonRunner[F] {
+  private val sync = Async[F]
+  private val files = Files[F]
+  implicit val andMonoid: Monoid[Boolean] = Monoid.instance[Boolean](true, _ && _)
+
+  private val executeBits = PosixPermissions(
+    OwnerExecute,
+    OwnerRead,
+    OwnerWrite,
+    GroupExecute,
+    GroupRead,
+    OthersExecute,
+    OthersRead
+  )
+
+  val runnerType = List(RunnerType.Trivial)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ) = {
+    val binaryPath = ctx.meta.runner
+      .flatMap(_.trivial)
+      .map(_.exec)
+      .map(bin => ctx.addonDir / bin)
+
+    binaryPath match {
+      case None =>
+        sync.raiseError(new IllegalStateException("No executable specified in addon!"))
+
+      case Some(file) =>
+        val bin = ctx.baseDir.relativize(file)
+        val cmd = SysCmd(bin.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env)
+
+        val withNSpawn =
+          OptionT
+            .whenF(cfg.nspawn.enabled)(RunnerUtil.checkContainer(logger, cfg.nspawn))
+            .getOrElse(false)
+
+        files.setPosixPermissions(file, executeBits).attempt *>
+          withNSpawn.flatMap {
+            case true =>
+              RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd)
+            case false =>
+              RunnerUtil.runDirectly(logger, ctx)(cmd)
+          }
+    }
+  }
+}
+
+object TrivialRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): TrivialRunner[F] =
+    new TrivialRunner[F](Config(cfg.nspawn, cfg.runTimeout))
+
+  case class Config(nspawn: NSpawn, timeout: Duration)
+}

modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.option._
+
+import docspell.common.UrlReader
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  tempDir.test("Read archive from directory") { dir =>
+    for {
+      archive <- IO(AddonArchive(dummyAddonUrl, "", ""))
+      path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir)
+
+      aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some)
+      _ = {
+        assertEquals(aa.name, "dummy-addon")
+        assertEquals(aa.version, "2.9")
+        assertEquals(aa.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+
+  test("Read archive from zip file") {
+    for {
+      archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO])
+      _ = {
+        assertEquals(archive.name, "dummy-addon")
+        assertEquals(archive.version, "2.9")
+        assertEquals(archive.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.logging.{Level, TestLoggingConfig}
+
+import munit._
+
+class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig {
+  val logger = docspell.logging.getLogger[IO]
+
+  override def docspellLogConfig =
+    super.docspellLogConfig.copy(minimumLevel = Level.Trace)
+
+  tempDir.test("select docker if Dockerfile exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake if flake.nix exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.NixFlake))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake and docker") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake))
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.common.Glob
+import docspell.files.Zip
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  test("read meta from zip file") {
+    val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192))
+    assertIO(meta, dummyAddonMeta)
+  }
+
+  tempDir.test("read meta from directory") { dir =>
+    for {
+      _ <- dummyAddonUrl
+        .readURL[IO](8192)
+        .through(Zip.unzip(8192, Glob.all))
+        .through(Zip.saveTo(logger, dir, moveUp = true))
+        .compile
+        .drain
+      meta <- AddonMeta.findInDirectory[IO](dir)
+      _ = assertEquals(meta, dummyAddonMeta)
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.addons.out.AddonOutput
+
+import io.circe.parser.decode
+import munit.FunSuite
+
+class AddonOutputTest extends FunSuite {
+
+  test("decode empty object") {
+    val out = decode[AddonOutput]("{}")
+    println(out)
+  }
+
+  test("decode sample output") {
+    val jsonStr =
+      """{ "files": [
+        |    {
+        |      "itemId": "qZDnyGIAJsXr",
+        |      "textFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.txt"
+        |      },
+        |      "pdfFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.pdf"
+        |      }
+        |    }
+        |  ]
+        |}
+        |""".stripMargin
+
+    val out = decode[AddonOutput](jsonStr)
+    println(out)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import cats.effect.IO
+import fs2.io.file.Path
+
+import docspell.common.LenientUri
+import docspell.common.exec.Env
+import docspell.logging.{Logger, TestLoggingConfig}
+
+import munit._
+
+class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig {
+
+  val logger = docspell.logging.getLogger[IO]
+
+  val dummyContext = Context(
+    addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""),
+    meta = AddonMeta.empty("test", "1.0"),
+    baseDir = Path(""),
+    addonDir = Path(""),
+    outputDir = Path(""),
+    cacheDir = Path("")
+  )
+
+  test("firstSuccessful must stop on first success") {
+    val counter = new AtomicInteger(0)
+    val runner = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(runner, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("firstSuccessful must try with next on error") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.failWith[IO]("failed")
+    val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(fail, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("do not retry on decoding errors") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(fail, increment, increment)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 0)
+    } yield ()
+  }
+
+  test("try on errors but stop on decoding error") {
+    val counter = new AtomicInteger(0)
+    val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val incrementFail =
+      new MockRunner(IO(counter.incrementAndGet()).void)
+        .as(AddonResult.executionFailed(new Exception("fail")))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(
+      incrementFail,
+      incrementFail,
+      decodeFail,
+      increment,
+      increment
+    )
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 2)
+    } yield ()
+  }
+
+  final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty)
+      extends AddonRunner[IO] {
+    val runnerType = Nil
+    def run(
+        logger: Logger[IO],
+        env: Env,
+        ctx: Context
+    ) = run.as(result)
+
+    def as(r: AddonResult) = new MockRunner(run, r)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.{Files, Path, PosixPermissions}
+
+import docspell.addons.AddonExecutorConfig._
+import docspell.addons.AddonMeta._
+import docspell.addons.AddonTriggerType._
+import docspell.common.{Duration, LenientUri}
+import docspell.logging.TestLoggingConfig
+
+import munit.CatsEffectSuite
+
+trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
+
+  val files: Files[IO] = Files[IO]
+
+  val dummyAddonUrl =
+    LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip"))
+
+  val dummyAddonMeta =
+    AddonMeta(
+      meta =
+        AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some),
+      triggers = Some(
+        Set(Scheduled, FinalProcessItem, FinalReprocessItem)
+      ),
+      None,
+      runner = Runner(
+        nix = NixRunner(true).some,
+        docker = DockerRunner(
+          enable = true,
+          image = None,
+          build = "Dockerfile".some
+        ).some,
+        trivial = TrivialRunner(true, "src/addon.sh").some
+      ).some,
+      options = Options(networking = true, collectOutput = true).some
+    )
+
+  def baseTempDir: Path =
+    Path(s"/tmp/target/test-temp")
+
+  val tempDir =
+    ResourceFixture[Path](
+      Resource.eval(Files[IO].createDirectories(baseTempDir)) *>
+        Files[IO]
+          .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777"))
+    )
+
+  def testExecutorConfig(
+      runner: RunnerType,
+      runners: RunnerType*
+  ): AddonExecutorConfig = {
+    val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100))
+    AddonExecutorConfig(
+      runner :: runners.toList,
+      Duration.minutes(2),
+      nspawn,
+      NixConfig("nix", Duration.minutes(2)),
+      DockerConfig("docker", Duration.minutes(2))
+    )
+  }
+}