Experiment with addons

Addons allow to execute external programs in some context inside
docspell. Currently it is possible to run them after processing files.
Addons are provided by URLs to zip files.

build.sbt

@@ -293,6 +293,15 @@ val openapiScalaSettings = Seq(
           field.copy(typeDef =
             TypeDef("DownloadState", Imports("docspell.common.DownloadState"))
           )
+      case "addon-trigger-type" =>
+        field =>
+          field.copy(typeDef =
+            TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType"))
+          )
+      case "addon-runner-type" =>
+        field =>
+          field
+            .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType")))
     })
 )
 
@@ -325,6 +334,7 @@ val common = project
     libraryDependencies ++=
       Dependencies.fs2 ++
         Dependencies.circe ++
+        Dependencies.circeGenericExtra ++
         Dependencies.calevCore ++
         Dependencies.calevCirce
   )
@@ -351,7 +361,7 @@ val files = project
   .in(file("modules/files"))
   .disablePlugins(RevolverPlugin)
   .settings(sharedSettings)
-  .withTestSettings
+  .withTestSettingsDependsOn(loggingScribe)
   .settings(
     name := "docspell-files",
     libraryDependencies ++=
@@ -448,6 +458,19 @@ val notificationApi = project
   )
   .dependsOn(common, loggingScribe)
 
+val addonlib = project
+  .in(file("modules/addonlib"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .withTestSettingsDependsOn(loggingScribe)
+  .settings(
+    libraryDependencies ++=
+      Dependencies.fs2 ++
+        Dependencies.circe ++
+        Dependencies.circeYaml
+  )
+  .dependsOn(common, files, loggingScribe)
+
 val store = project
   .in(file("modules/store"))
   .disablePlugins(RevolverPlugin)
@@ -469,7 +492,16 @@ val store = project
     libraryDependencies ++=
       Dependencies.testContainer.map(_ % Test)
   )
-  .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe)
+  .dependsOn(
+    common,
+    addonlib,
+    query.jvm,
+    totp,
+    files,
+    notificationApi,
+    jsonminiq,
+    loggingScribe
+  )
 
 val notificationImpl = project
   .in(file("modules/notification/impl"))
@@ -647,7 +679,7 @@ val restapi = project
     openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, query.jvm, notificationApi, jsonminiq)
+  .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib)
 
 val joexapi = project
   .in(file("modules/joexapi"))
@@ -667,7 +699,7 @@ val joexapi = project
     openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, loggingScribe)
+  .dependsOn(common, loggingScribe, addonlib)
 
 val backend = project
   .in(file("modules/backend"))
@@ -683,6 +715,7 @@ val backend = project
         Dependencies.emil
   )
   .dependsOn(
+    addonlib,
     store,
     notificationApi,
     joexapi,
@@ -739,7 +772,7 @@ val config = project
       Dependencies.fs2 ++
         Dependencies.pureconfig
   )
-  .dependsOn(common, loggingApi, ftspsql, store)
+  .dependsOn(common, loggingApi, ftspsql, store, addonlib)
 
 // --- Application(s)
 
@@ -946,6 +979,7 @@ val root = project
   )
   .aggregate(
     common,
+    addonlib,
     loggingApi,
     loggingScribe,
     config,

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.Zip
+
+final case class AddonArchive(url: LenientUri, name: String, version: String) {
+  def nameAndVersion: String =
+    s"$name-$version"
+
+  def extractTo[F[_]: Async](
+      reader: UrlReader[F],
+      directory: Path,
+      withSubdir: Boolean = true,
+      glob: Glob = Glob.all
+  ): F[Path] = {
+    val logger = docspell.logging.getLogger[F]
+    val target =
+      if (withSubdir) directory.absolute / nameAndVersion
+      else directory.absolute
+
+    Files[F]
+      .exists(target)
+      .flatMap {
+        case true => target.pure[F]
+        case false =>
+          Files[F].createDirectories(target) *>
+            reader(url)
+              .through(Zip.unzip(8192, glob))
+              .through(Zip.saveTo(logger, target, moveUp = true))
+              .compile
+              .drain
+              .as(target)
+      }
+  }
+
+  /** Read meta either from the given directory or extract the url to find the metadata
+    * file to read
+    */
+  def readMeta[F[_]: Async](
+      urlReader: UrlReader[F],
+      directory: Option[Path] = None
+  ): F[AddonMeta] =
+    directory
+      .map(AddonMeta.findInDirectory[F])
+      .getOrElse(AddonMeta.findInZip(urlReader(url)))
+}
+
+object AddonArchive {
+  def read[F[_]: Async](
+      url: LenientUri,
+      urlReader: UrlReader[F],
+      extractDir: Option[Path] = None
+  ): F[AddonArchive] = {
+    val addon = AddonArchive(url, "", "")
+    addon
+      .readMeta(urlReader, extractDir)
+      .map(m => addon.copy(name = m.meta.name, version = m.meta.version))
+  }
+
+  def dockerAndFlakeExists[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[(Boolean, Boolean)] = {
+    val files = Files[F]
+    def forPath(path: Path): F[(Boolean, Boolean)] =
+      (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled
+
+    def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] =
+      data
+        .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix")))
+        .collect {
+          case bin if bin.name == "Dockerfile" => (true, false)
+          case bin if bin.name == "flake.nix"  => (false, true)
+        }
+        .compile
+        .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2))
+
+    archive.fold(forPath, forZip)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+import cats.syntax.all._
+
+case class AddonExecutionResult(
+    addonResults: List[AddonResult],
+    pure: Boolean
+) {
+  def addonResult: AddonResult = addonResults.combineAll
+  def isFailure: Boolean = addonResult.isFailure
+  def isSuccess: Boolean = addonResult.isSuccess
+}
+
+object AddonExecutionResult {
+  val empty: AddonExecutionResult =
+    AddonExecutionResult(Nil, false)
+
+  def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult =
+    AddonExecutionResult(
+      a.addonResults ::: b.addonResults,
+      a.pure && b.pure
+    )
+
+  implicit val executionResultMonoid: Monoid[AddonExecutionResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.Kleisli
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file._
+
+import docspell.common.UrlReader
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonExecutor[F[_]] {
+
+  def config: AddonExecutorConfig
+
+  def execute(logger: Logger[F]): AddonExec[F]
+
+  def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] =
+    execute(logger).run(in)
+}
+
+object AddonExecutor {
+
+  def apply[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      urlReader: UrlReader[F]
+  ): AddonExecutor[F] =
+    new AddonExecutor[F] with AddonLoggerExtension {
+      val config = cfg
+
+      def execute(logger: Logger[F]): AddonExec[F] =
+        Kleisli { in =>
+          for {
+            _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}")
+            ctx <- prepareDirectory(
+              logger,
+              in.baseDir,
+              in.outputDir,
+              in.cacheDir,
+              in.addons
+            )
+            rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c))
+            pure = ctx.foldl(true)((b, c) => b && c.meta.isPure)
+          } yield AddonExecutionResult(rs, pure)
+        }
+
+      private def prepareDirectory(
+          logger: Logger[F],
+          baseDir: Path,
+          outDir: Path,
+          cacheDir: Path,
+          addons: List[AddonRef]
+      ): F[List[Context]] =
+        for {
+          addonsDir <- Directory.create(baseDir / "addons")
+          _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir)
+          _ <- Context
+            .userInputFile(baseDir)
+            .parent
+            .fold(().pure[F])(Files[F].createDirectories)
+          archives = addons.map(_.archive).distinctBy(_.url)
+          _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir")
+          mkCtxs <- archives.traverse { archive =>
+            for {
+              _ <- logger.debug(s"Extracting $archive")
+              addonDir <- archive.extractTo(urlReader, addonsDir)
+              meta <- AddonMeta.findInDirectory(addonDir)
+              mkCtx = (ref: AddonRef) =>
+                Context(ref, meta, baseDir, addonDir, outDir, cacheDir)
+            } yield archive.url -> mkCtx
+          }
+          ctxFactory = mkCtxs.toMap
+          res = addons.map(ref => ctxFactory(ref.archive.url)(ref))
+        } yield res
+
+      private def runAddon(logger: Logger[F], env: Env)(
+          ctx: Context
+      ): F[AddonResult] =
+        for {
+          _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}")
+          _ <- logger.trace("Storing user input into file")
+          _ <- Stream
+            .emit(ctx.addon.args)
+            .through(fs2.text.utf8.encode)
+            .through(Files[F].writeAll(ctx.userInputFile, Flags.Write))
+            .compile
+            .drain
+
+          runner <- selectRunner(cfg, ctx.meta, ctx.addonDir)
+          result <- runner.run(logger, env, ctx)
+        } yield result
+    }
+
+  def selectRunner[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      meta: AddonMeta,
+      addonDir: Path
+  ): F[AddonRunner[F]] =
+    for {
+      addonRunner <- meta.enabledTypes(Left(addonDir))
+      // intersect on list retains order in first
+      possibleRunner = cfg.runner
+        .intersect(addonRunner)
+        .map(AddonRunner.forType[F](cfg))
+      runner = possibleRunner match {
+        case Nil =>
+          AddonRunner.failWith(
+            s"No runner available for addon config ${meta.runner} and config ${cfg.runner}."
+          )
+        case list =>
+          AddonRunner.firstSuccessful(list)
+      }
+    } yield runner
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.addons.AddonExecutorConfig._
+import docspell.common.Duration
+import docspell.common.exec.{Args, SysCmd}
+
+case class AddonExecutorConfig(
+    runner: List[RunnerType],
+    runTimeout: Duration,
+    nspawn: NSpawn,
+    nixRunner: NixConfig,
+    dockerRunner: DockerConfig
+)
+
+object AddonExecutorConfig {
+
+  case class NSpawn(
+      enabled: Boolean,
+      sudoBinary: String,
+      nspawnBinary: String,
+      containerWait: Duration
+  ) {
+    val nspawnVersion =
+      SysCmd(nspawnBinary, Args.of("--version")).withTimeout(Duration.seconds(2))
+  }
+
+  case class NixConfig(
+      nixBinary: String,
+      buildTimeout: Duration
+  )
+
+  case class DockerConfig(
+      dockerBinary: String,
+      buildTimeout: Duration
+  ) {
+    def dockerBuild(imageName: String): SysCmd =
+      SysCmd(dockerBinary, "build", "-t", imageName, ".").withTimeout(buildTimeout)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.logging.Logger
+
+trait AddonLoggerExtension {
+
+  implicit final class LoggerAddonOps[F[_]](self: Logger[F]) {
+    private val addonName = "addon-name"
+    private val addonVersion = "addon-version"
+
+    def withAddon(r: AddonArchive): Logger[F] =
+      self.capture(addonName, r.name).capture(addonVersion, r.version)
+
+    def withAddon(r: Context): Logger[F] =
+      withAddon(r.addon.archive)
+
+    def withAddon(m: AddonMeta): Logger[F] =
+      self.capture(addonName, m.meta.name).capture(addonVersion, m.meta.version)
+  }
+}
+
+object AddonLoggerExtension extends AddonLoggerExtension

modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala

@@ -0,0 +1,216 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import java.io.FileNotFoundException
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common.Glob
+import docspell.files.Zip
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.yaml.{parser => YamlParser}
+import io.circe.{Decoder, Encoder}
+import io.circe.{parser => JsonParser}
+
+case class AddonMeta(
+    meta: AddonMeta.Meta,
+    triggers: Option[Set[AddonTriggerType]],
+    args: Option[List[String]],
+    runner: Option[AddonMeta.Runner],
+    options: Option[AddonMeta.Options]
+) {
+
+  def nameAndVersion: String =
+    s"${meta.name}-${meta.version}"
+
+  def parseResult: Boolean =
+    options.exists(_.collectOutput)
+
+  def ignoreResult: Boolean =
+    !parseResult
+
+  def isImpure: Boolean =
+    options.exists(_.isImpure)
+
+  def isPure: Boolean =
+    options.forall(_.isPure)
+
+  /** Returns a list of runner types that are possible to use for this addon. This is also
+    * inspecting the archive to return defaults when the addon isn't declaring it in the
+    * descriptor.
+    */
+  def enabledTypes[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[List[RunnerType]] =
+    for {
+      filesExists <- AddonArchive.dockerAndFlakeExists(archive)
+      (dockerFileExists, flakeFileExists) = filesExists
+
+      nixEnabled = runner.flatMap(_.nix).map(_.enable) match {
+        case Some(flag) => flag
+        case None       => flakeFileExists
+      }
+
+      dockerEnabled = runner.flatMap(_.docker).map(_.enable) match {
+        case Some(flag) => flag
+        case None       => dockerFileExists
+      }
+
+      trivialEnabled = runner.flatMap(_.trivial).exists(_.enable)
+
+      result = RunnerType.all.filter(_.fold(nixEnabled, dockerEnabled, trivialEnabled))
+    } yield result
+
+}
+
+object AddonMeta {
+
+  def empty(name: String, version: String): AddonMeta =
+    AddonMeta(Meta(name, version, None), None, None, None, None)
+
+  case class Meta(name: String, version: String, description: Option[String])
+  case class Runner(
+      nix: Option[NixRunner],
+      docker: Option[DockerRunner],
+      trivial: Option[TrivialRunner]
+  )
+  case class NixRunner(enable: Boolean)
+  case class DockerRunner(enable: Boolean, image: Option[String], build: Option[String])
+  case class TrivialRunner(enable: Boolean, exec: String)
+  case class Options(networking: Boolean, collectOutput: Boolean) {
+    def isPure = !networking && collectOutput
+    def isImpure = networking
+    def isUseless = !networking && !collectOutput
+    def isUseful = networking || collectOutput
+  }
+
+  object NixRunner {
+    implicit val jsonEncoder: Encoder[NixRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[NixRunner] =
+      deriveDecoder
+  }
+
+  object DockerRunner {
+    implicit val jsonEncoder: Encoder[DockerRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[DockerRunner] =
+      deriveDecoder
+  }
+
+  object TrivialRunner {
+    implicit val jsonEncoder: Encoder[TrivialRunner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[TrivialRunner] =
+      deriveDecoder
+  }
+
+  object Runner {
+    implicit val jsonEncoder: Encoder[Runner] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Runner] =
+      deriveDecoder
+  }
+
+  object Options {
+    implicit val jsonEncoder: Encoder[Options] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Options] =
+      deriveDecoder
+  }
+
+  object Meta {
+    implicit val jsonEncoder: Encoder[Meta] =
+      deriveEncoder
+    implicit val jsonDecoder: Decoder[Meta] =
+      deriveDecoder
+  }
+
+  implicit val jsonEncoder: Encoder[AddonMeta] =
+    deriveEncoder
+
+  implicit val jsonDecoder: Decoder[AddonMeta] =
+    deriveDecoder
+
+  def fromJsonString(str: String): Either[Throwable, AddonMeta] =
+    JsonParser.decode[AddonMeta](str)
+
+  def fromJsonBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] =
+    bytes
+      .through(fs2.text.utf8.decode)
+      .compile
+      .string
+      .map(fromJsonString)
+      .rethrow
+
+  def fromYamlString(str: String): Either[Throwable, AddonMeta] =
+    YamlParser.parse(str).flatMap(_.as[AddonMeta])
+
+  def fromYamlBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] =
+    bytes
+      .through(fs2.text.utf8.decode)
+      .compile
+      .string
+      .map(fromYamlString)
+      .rethrow
+
+  def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = {
+    val logger = docspell.logging.getLogger[F]
+    val jsonFile = dir / "docspell-addon.json"
+    val yamlFile = dir / "docspell-addon.yaml"
+    val yamlFile2 = dir / "docspell-addon.yml"
+
+    OptionT
+      .liftF(Files[F].exists(jsonFile))
+      .flatTap(OptionT.whenF(_)(logger.debug(s"Reading json addon file $jsonFile")))
+      .flatMap(OptionT.whenF(_)(fromJsonBytes(Files[F].readAll(jsonFile))))
+      .orElse(
+        OptionT
+          .liftF(Files[F].exists(yamlFile))
+          .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile")))
+          .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile))))
+      )
+      .orElse(
+        OptionT
+          .liftF(Files[F].exists(yamlFile2))
+          .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile2")))
+          .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile2))))
+      )
+      .getOrElseF(
+        Sync[F].raiseError(
+          new FileNotFoundException(s"No docspell-addon.{yaml|json} file found in $dir!")
+        )
+      )
+  }
+
+  def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = {
+    val fail: F[AddonMeta] = Async[F].raiseError(
+      new FileNotFoundException(
+        s"No docspell-addon.{yaml|json} file found in zip!"
+      )
+    )
+    zipFile
+      .through(Zip.unzip(8192, Glob("**/docspell-addon.*")))
+      .filter(bin => !bin.name.endsWith("/"))
+      .flatMap { bin =>
+        if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data))
+        else if (bin.extensionIn(Set("yaml", "yml")))
+          Stream.eval(AddonMeta.fromYamlBytes(bin.data))
+        else Stream.empty
+      }
+      .take(1)
+      .compile
+      .last
+      .flatMap(_.map(Sync[F].pure).getOrElse(fail))
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+case class AddonRef(archive: AddonArchive, args: String)

modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+
+import docspell.addons.out.AddonOutput
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Codec, Decoder, Encoder}
+
+sealed trait AddonResult {
+  def toEither: Either[Throwable, AddonOutput]
+
+  def isSuccess: Boolean = toEither.isRight
+  def isFailure: Boolean = !isSuccess
+
+  def cast: AddonResult = this
+}
+
+object AddonResult {
+
+  /** The addon was run successful, but decoding its stdout failed. */
+  case class DecodingError(message: String) extends AddonResult {
+    def toEither = Left(new IllegalStateException(message))
+  }
+  object DecodingError {
+    implicit val jsonEncoder: Encoder[DecodingError] = deriveEncoder
+    implicit val jsonDecoder: Decoder[DecodingError] = deriveDecoder
+  }
+
+  def decodingError(message: String): AddonResult =
+    DecodingError(message)
+
+  def decodingError(ex: Throwable): AddonResult =
+    DecodingError(ex.getMessage)
+
+  /** Running the addon resulted in an invalid return code (!= 0). */
+  case class ExecutionError(rc: Int) extends AddonResult {
+    def toEither = Left(new IllegalStateException(s"Exit code: $rc"))
+  }
+
+  object ExecutionError {
+    implicit val jsonEncoder: Encoder[ExecutionError] = deriveEncoder
+    implicit val jsonDecoder: Decoder[ExecutionError] = deriveDecoder
+  }
+
+  def executionError(rc: Int): AddonResult =
+    ExecutionError(rc)
+
+  /** The execution of the addon failed with an exception. */
+  case class ExecutionFailed(error: Throwable) extends AddonResult {
+    def toEither = Left(error)
+  }
+
+  object ExecutionFailed {
+    implicit val throwableCodec: Codec[Throwable] =
+      Codec.from(
+        Decoder[String].emap(str => Right(ErrorMessageThrowable(str))),
+        Encoder[String].contramap(_.getMessage)
+      )
+
+    implicit val jsonEncoder: Encoder[ExecutionFailed] = deriveEncoder
+    implicit val jsonDecoder: Decoder[ExecutionFailed] = deriveDecoder
+
+    private class ErrorMessageThrowable(msg: String) extends RuntimeException(msg) {
+      override def fillInStackTrace() = this
+    }
+    private object ErrorMessageThrowable {
+      def apply(str: String): Throwable = new ErrorMessageThrowable(str)
+    }
+  }
+
+  def executionFailed(error: Throwable): AddonResult =
+    ExecutionFailed(error)
+
+  /** The addon was run successfully and its output was decoded (if any). */
+  case class Success(output: AddonOutput) extends AddonResult {
+    def toEither = Right(output)
+  }
+
+  object Success {
+    implicit val jsonEncoder: Encoder[Success] = deriveEncoder
+    implicit val jsonDecoder: Decoder[Success] = deriveDecoder
+  }
+
+  def success(output: AddonOutput): AddonResult =
+    Success(output)
+
+  val empty: AddonResult = Success(AddonOutput.empty)
+
+  def combine(a: AddonResult, b: AddonResult): AddonResult =
+    (a, b) match {
+      case (Success(o1), Success(o2)) => Success(AddonOutput.combine(o1, o2))
+      case (Success(_), e)            => e
+      case (e, Success(_))            => e
+      case _                          => a
+    }
+
+  implicit val deriveConfig: Configuration =
+    Configuration.default.withDiscriminator("result").withKebabCaseConstructorNames
+
+  implicit val jsonDecoder: Decoder[AddonResult] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AddonResult] = deriveConfiguredEncoder
+
+  implicit val addonResultMonoid: Monoid[AddonResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Applicative
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+
+import docspell.addons.runner._
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonRunner[F[_]] {
+  def runnerType: List[RunnerType]
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ): F[AddonResult]
+}
+
+object AddonRunner {
+  def forType[F[_]: Async](cfg: AddonExecutorConfig)(rt: RunnerType) =
+    rt match {
+      case RunnerType.NixFlake => NixFlakeRunner[F](cfg)
+      case RunnerType.Docker   => DockerRunner[F](cfg)
+      case RunnerType.Trivial  => TrivialRunner[F](cfg)
+    }
+
+  def failWith[F[_]](errorMsg: String)(implicit F: Applicative[F]): AddonRunner[F] =
+    pure(AddonResult.executionFailed(new Exception(errorMsg)))
+
+  def pure[F[_]: Applicative](result: AddonResult): AddonRunner[F] =
+    new AddonRunner[F] {
+      val runnerType = Nil
+
+      def run(logger: Logger[F], env: Env, ctx: Context) =
+        Applicative[F].pure(result)
+    }
+
+  def firstSuccessful[F[_]: Sync](runners: List[AddonRunner[F]]): AddonRunner[F] =
+    runners match {
+      case Nil      => failWith("No runner available!")
+      case a :: Nil => a
+      case _ =>
+        new AddonRunner[F] {
+          val runnerType = runners.flatMap(_.runnerType).distinct
+
+          def run(logger: Logger[F], env: Env, ctx: Context) =
+            Stream
+              .emits(runners)
+              .evalTap(r =>
+                logger.info(
+                  s"Attempt to run addon ${ctx.meta.nameAndVersion} with runner ${r.runnerType}"
+                )
+              )
+              .evalMap(_.run(logger, env, ctx))
+              .flatMap {
+                case r @ AddonResult.Success(_) => Stream.emit(r.cast.some)
+                case r @ AddonResult.ExecutionFailed(ex) =>
+                  if (ctx.meta.isPure) {
+                    logger.stream
+                      .warn(ex)(s"Addon runner failed, try next.")
+                      .as(r.cast.some)
+                  } else {
+                    logger.stream.warn(ex)(s"Addon runner failed!").as(None)
+                  }
+                case r @ AddonResult.ExecutionError(rc) =>
+                  if (ctx.meta.isPure) {
+                    logger.stream
+                      .warn(s"Addon runner returned non-zero: $rc. Try next.")
+                      .as(r.cast.some)
+                  } else {
+                    logger.stream.warn(s"Addon runner returned non-zero: $rc!").as(None)
+                  }
+                case AddonResult.DecodingError(message) =>
+                  // Don't retry as it is very unlikely that the output differs using another runner
+                  // This is most likely a bug in the addon
+                  logger.stream
+                    .warn(
+                      s"Error decoding the output of the addon ${ctx.meta.nameAndVersion}: $message. Stopping here. This is likely a bug in the addon."
+                    )
+                    .as(None)
+              }
+              .unNoneTerminate
+              .takeThrough(_.isFailure)
+              .compile
+              .last
+              .flatMap {
+                case Some(r) => r.pure[F]
+                case None =>
+                  AddonResult
+                    .executionFailed(new NoSuchElementException("No runner left :("))
+                    .pure[F]
+              }
+        }
+    }
+
+  def firstSuccessful[F[_]: Sync](
+      runner: AddonRunner[F],
+      runners: AddonRunner[F]*
+  ): AddonRunner[F] =
+    firstSuccessful(runner :: runners.toList)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.NonEmptyList
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait AddonTriggerType {
+  def name: String
+}
+
+object AddonTriggerType {
+
+  /** The final step when processing an item. */
+  case object FinalProcessItem extends AddonTriggerType {
+    val name = "final-process-item"
+  }
+
+  /** The final step when reprocessing an item. */
+  case object FinalReprocessItem extends AddonTriggerType {
+    val name = "final-reprocess-item"
+  }
+
+  /** Running periodically based on a schedule. */
+  case object Scheduled extends AddonTriggerType {
+    val name = "scheduled"
+  }
+
+  /** Running (manually) on some existing item. */
+  case object ExistingItem extends AddonTriggerType {
+    val name = "existing-item"
+  }
+
+  val all: NonEmptyList[AddonTriggerType] =
+    NonEmptyList.of(FinalProcessItem, FinalReprocessItem, Scheduled, ExistingItem)
+
+  def fromString(str: String): Either[String, AddonTriggerType] =
+    all
+      .find(e => e.name.equalsIgnoreCase(str))
+      .toRight(s"Invalid addon trigger type: $str")
+
+  def unsafeFromString(str: String): AddonTriggerType =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonEncoder: Encoder[AddonTriggerType] =
+    Encoder.encodeString.contramap(_.name)
+
+  implicit val jsonDecoder: Decoder[AddonTriggerType] =
+    Decoder.decodeString.emap(fromString)
+}

modules/addonlib/src/main/scala/docspell/addons/Context.scala

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import fs2.io.file.Path
+
+import docspell.common._
+import docspell.common.exec.{Args, Env, SysCmd}
+
+/** Context a list of addons is executed in.
+  *
+  * Each addon has its own `addonDir`, but all share the same `baseDir` in one run.
+  */
+case class Context(
+    addon: AddonRef,
+    meta: AddonMeta,
+    baseDir: Path,
+    addonDir: Path,
+    outputDir: Path,
+    cacheDir: Path
+) {
+  def userInputFile = Context.userInputFile(baseDir)
+  def tempDir = Context.tempDir(baseDir)
+
+  private[addons] def addonCommand(
+      binary: String,
+      timeout: Duration,
+      relativeToBase: Boolean,
+      outputDir: Option[String],
+      cacheDir: Option[String]
+  ): SysCmd = {
+    val execBin = Option
+      .when(relativeToBase)(binary)
+      .getOrElse((baseDir / binary).toString)
+
+    val input = Option
+      .when(relativeToBase)(baseDir.relativize(userInputFile))
+      .getOrElse(userInputFile)
+
+    val allArgs =
+      Args(meta.args.getOrElse(Nil)).append(input)
+    val envAddonDir = Option
+      .when(relativeToBase)(baseDir.relativize(addonDir))
+      .getOrElse(addonDir)
+    val envTmpDir = Option
+      .when(relativeToBase)(baseDir.relativize(tempDir))
+      .getOrElse(tempDir)
+    val outDir = outputDir.getOrElse(this.outputDir.toString)
+    val cache = cacheDir.getOrElse(this.cacheDir.toString)
+    val moreEnv =
+      Env.of(
+        "ADDON_DIR" -> envAddonDir.toString,
+        "TMPDIR" -> envTmpDir.toString,
+        "TMP_DIR" -> envTmpDir.toString,
+        "OUTPUT_DIR" -> outDir,
+        "CACHE_DIR" -> cache
+      )
+
+    SysCmd(execBin, allArgs).withTimeout(timeout).addEnv(moreEnv)
+  }
+}
+
+object Context {
+  def userInputFile(base: Path): Path =
+    base / "arguments" / "user-input"
+  def tempDir(base: Path): Path =
+    base / "temp"
+}

modules/addonlib/src/main/scala/docspell/addons/Directory.scala

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import cats.{Applicative, Monad}
+import fs2.io.file.{Files, Path, PosixPermissions}
+
+object Directory {
+
+  def create[F[_]: Files: Applicative](dir: Path): F[Path] =
+    Files[F]
+      .createDirectories(dir, PosixPermissions.fromOctal("777"))
+      .as(dir)
+
+  def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] =
+    (dir :: dirs.toList).traverse_(Files[F].createDirectories(_))
+
+  def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
+    List(
+      Files[F].isDirectory(dir),
+      Files[F].list(dir).take(1).compile.last.map(_.isDefined)
+    ).sequence.map(_.forall(identity))
+
+  def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
+    nonEmpty(dir).map(b => !b)
+
+  def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
+    for {
+      _ <- Resource.eval(Files[F].createDirectories(parent))
+      d <- mkTemp(parent, prefix)
+    } yield d
+
+  def temp2[F[_]: Files](
+      parent: Path,
+      prefix1: String,
+      prefix2: String
+  ): Resource[F, (Path, Path)] =
+    for {
+      _ <- Resource.eval(Files[F].createDirectories(parent))
+      a <- mkTemp(parent, prefix1)
+      b <- mkTemp(parent, prefix2)
+    } yield (a, b)
+
+  def createTemp[F[_]: Files: Monad](
+      parent: Path,
+      prefix: String
+  ): F[Path] =
+    for {
+      _ <- Files[F].createDirectories(parent)
+      d <- mkTemp_(parent, prefix)
+    } yield d
+
+  private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
+    Files[F]
+      .tempDirectory(
+        parent.some,
+        prefix,
+        PosixPermissions.fromOctal("777")
+      )
+
+  private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] =
+    Files[F]
+      .createTempDirectory(
+        parent.some,
+        prefix,
+        PosixPermissions.fromOctal("777")
+      )
+}

modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect.Resource
+import fs2.io.file.{Files, Path}
+
+import docspell.common.exec.Env
+
+case class InputEnv(
+    addons: List[AddonRef],
+    baseDir: Path,
+    outputDir: Path,
+    cacheDir: Path,
+    env: Env
+) {
+  def addEnv(key: String, value: String): InputEnv =
+    copy(env = env.add(key, value))
+
+  def addEnv(vp: (String, String)*): InputEnv =
+    copy(env = env.addAll(vp.toMap))
+
+  def addEnv(vm: Map[String, String]): InputEnv =
+    copy(env = env ++ Env(vm))
+
+  def withTempBase[F[_]: Files]: Resource[F, InputEnv] =
+    Directory.temp(baseDir, "addon-").map(path => copy(baseDir = path))
+}

modules/addonlib/src/main/scala/docspell/addons/Middleware.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monad
+import cats.data.Kleisli
+import cats.effect.kernel.Sync
+import cats.syntax.all._
+import fs2.io.file.Files
+
+trait Middleware[F[_]] extends (AddonExec[F] => AddonExec[F]) { self =>
+
+  def >>(next: Middleware[F]): Middleware[F] =
+    Middleware(self.andThen(next))
+}
+
+object Middleware {
+  def apply[F[_]](f: AddonExec[F] => AddonExec[F]): Middleware[F] =
+    a => f(a)
+
+  def identity[F[_]]: Middleware[F] = Middleware(scala.Predef.identity)
+
+  /** Uses a temporary base dir that is removed after execution. Use this as the last
+    * layer!
+    */
+  def ephemeralRun[F[_]: Files: Sync]: Middleware[F] =
+    Middleware(a => Kleisli(_.withTempBase.use(a.run)))
+
+  /** Prepare running an addon */
+  def prepare[F[_]: Monad](
+      prep: Kleisli[F, InputEnv, InputEnv]
+  ): Middleware[F] =
+    Middleware(a => Kleisli(in => prep.run(in).flatMap(a.run)))
+
+  def postProcess[F[_]: Monad](
+      post: Kleisli[F, AddonExecutionResult, Unit]
+  ): Middleware[F] =
+    Middleware(_.flatMapF(r => post.map(_ => r).run(r)))
+}

modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.NonEmptyList
+import cats.syntax.all._
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait RunnerType {
+  def name: String
+
+  def fold[A](
+      nixFlake: => A,
+      docker: => A,
+      trivial: => A
+  ): A
+}
+object RunnerType {
+  case object NixFlake extends RunnerType {
+    val name = "nix-flake"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = nixFlake
+  }
+  case object Docker extends RunnerType {
+    val name = "docker"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = docker
+  }
+  case object Trivial extends RunnerType {
+    val name = "trivial"
+
+    def fold[A](
+        nixFlake: => A,
+        docker: => A,
+        trivial: => A
+    ): A = trivial
+  }
+
+  val all: NonEmptyList[RunnerType] =
+    NonEmptyList.of(NixFlake, Docker, Trivial)
+
+  def fromString(str: String): Either[String, RunnerType] =
+    all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid runner value: $str")
+
+  def unsafeFromString(str: String): RunnerType =
+    fromString(str).fold(sys.error, identity)
+
+  def fromSeparatedString(str: String): Either[String, List[RunnerType]] =
+    str.split("[\\s,]+").toList.map(_.trim).traverse(fromString)
+
+  implicit val jsonDecoder: Decoder[RunnerType] =
+    Decoder[String].emap(RunnerType.fromString)
+
+  implicit val jsonEncoder: Encoder[RunnerType] =
+    Encoder[String].contramap(_.name)
+}

modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.kernel.Monoid
+
+import docspell.common.bc.BackendCommand
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Decoded stdout result from executing an addon. */
+case class AddonOutput(
+    commands: List[BackendCommand] = Nil,
+    files: List[ItemFile] = Nil,
+    newItems: List[NewItem] = Nil
+)
+
+object AddonOutput {
+  val empty: AddonOutput = AddonOutput()
+
+  def combine(a: AddonOutput, b: AddonOutput): AddonOutput =
+    AddonOutput(a.commands ++ b.commands, a.files ++ b.files)
+
+  implicit val addonResultMonoid: Monoid[AddonOutput] =
+    Monoid.instance(empty, combine)
+
+  implicit val jsonConfig: Configuration =
+    Configuration.default.withDefaults
+
+  implicit val jsonDecoder: Decoder[AddonOutput] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AddonOutput] = deriveConfiguredEncoder
+
+  def fromString(str: String): Either[Throwable, AddonOutput] =
+    io.circe.parser.decode[AddonOutput](str)
+
+  def unsafeFromString(str: String): AddonOutput =
+    fromString(str).fold(throw _, identity)
+}

modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.FileSupport._
+import docspell.logging.Logger
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Addons can produce files in their output directory. These can be named here in order
+  * to do something with them.
+  *
+  *   - textFiles will replace the extracted text with the contents of the file
+  *   - pdfFiles will add/replace the converted pdf with the given file
+  *   - previewImages will add/replace preview images
+  *   - newFiles will be added as new attachments to the item
+  *
+  * Files must be referenced by attachment id.
+  */
+final case class ItemFile(
+    itemId: Ident,
+    textFiles: Map[String, String] = Map.empty,
+    pdfFiles: Map[String, String] = Map.empty,
+    previewImages: Map[String, String] = Map.empty,
+    newFiles: List[NewFile] = Nil
+) {
+  def isEmpty: Boolean =
+    textFiles.isEmpty && pdfFiles.isEmpty && previewImages.isEmpty
+
+  def nonEmpty: Boolean = !isEmpty
+
+  def resolveTextFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.text("*"), textFiles)
+
+  def resolvePdfFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.pdf, pdfFiles)
+
+  def resolvePreviewFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(String, Path)]] =
+    resolveFiles(logger, outputDir, MimeType.image("*"), previewImages)
+
+  def resolveNewFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[(NewFile, Path)]] =
+    newFiles.traverseFilter(nf =>
+      nf.resolveFile(logger, outputDir).map(_.map(p => (nf, p)))
+    )
+
+  private def resolveFiles[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path,
+      mime: MimeType,
+      files: Map[String, String]
+  ): F[List[(String, Path)]] = {
+    val allFiles =
+      files.toList.map(t => t._1 -> outputDir / t._2)
+
+    allFiles.traverseFilter { case (key, file) =>
+      OptionT(file.detectMime)
+        .flatMapF(fileType =>
+          if (mime.matches(fileType)) (key -> file).some.pure[F]
+          else
+            logger
+              .warn(
+                s"File $file provided as ${mime.asString} file, but was recognized as ${fileType.asString}. Ignoring it."
+              )
+              .as(None: Option[(String, Path)])
+        )
+        .value
+    }
+  }
+}
+
+object ItemFile {
+
+  implicit val jsonConfig: Configuration =
+    Configuration.default.withDefaults
+
+  implicit val jsonEncoder: Encoder[ItemFile] = deriveConfiguredEncoder
+  implicit val jsonDecoder: Decoder[ItemFile] = deriveConfiguredDecoder
+}

modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.effect.Sync
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.out.NewFile.Meta
+import docspell.common.ProcessItemArgs.ProcessMeta
+import docspell.common.{Ident, Language}
+import docspell.logging.Logger
+
+import io.circe.Codec
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.deriveConfiguredCodec
+import io.circe.generic.semiauto.deriveCodec
+
+case class NewFile(metadata: Meta = Meta.empty, file: String) {
+
+  def resolveFile[F[_]: Files: Sync](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[Option[Path]] = {
+    val target = outputDir / file
+    Files[F]
+      .exists(target)
+      .flatMap(flag =>
+        if (flag) target.some.pure[F]
+        else logger.warn(s"File not found: $file").as(Option.empty)
+      )
+  }
+}
+
+object NewFile {
+
+  case class Meta(
+      language: Option[Language],
+      skipDuplicate: Option[Boolean],
+      attachmentsOnly: Option[Boolean]
+  ) {
+
+    def toProcessMeta(
+        cid: Ident,
+        itemId: Ident,
+        collLang: Option[Language],
+        sourceAbbrev: String
+    ): ProcessMeta =
+      ProcessMeta(
+        collective = cid,
+        itemId = Some(itemId),
+        language = language.orElse(collLang).getOrElse(Language.English),
+        direction = None,
+        sourceAbbrev = sourceAbbrev,
+        folderId = None,
+        validFileTypes = Seq.empty,
+        skipDuplicate = skipDuplicate.getOrElse(true),
+        fileFilter = None,
+        tags = None,
+        reprocess = false,
+        attachmentsOnly = attachmentsOnly
+      )
+  }
+
+  object Meta {
+    val empty = Meta(None, None, None)
+    implicit val jsonCodec: Codec[Meta] = deriveCodec
+  }
+
+  implicit val jsonConfig: Configuration = Configuration.default.withDefaults
+
+  implicit val jsonCodec: Codec[NewFile] = deriveConfiguredCodec
+}

modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.out
+
+import cats.Monad
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.out.NewItem.Meta
+import docspell.common._
+import docspell.logging.Logger
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+case class NewItem(metadata: Option[Meta], files: List[String]) {
+
+  def toProcessMeta(
+      cid: Ident,
+      collLang: Option[Language],
+      sourceAbbrev: String
+  ): ProcessItemArgs.ProcessMeta =
+    metadata
+      .getOrElse(Meta(None, None, None, None, None, None, None))
+      .toProcessArgs(cid, collLang, sourceAbbrev)
+
+  def resolveFiles[F[_]: Files: Monad](
+      logger: Logger[F],
+      outputDir: Path
+  ): F[List[Path]] = {
+    val allFiles =
+      files.map(name => outputDir / name)
+
+    allFiles.traverseFilter { file =>
+      Files[F]
+        .exists(file)
+        .flatMap {
+          case true => file.some.pure[F]
+          case false =>
+            logger
+              .warn(s"File $file doesn't exist. Ignoring it.")
+              .as(None)
+        }
+    }
+  }
+}
+
+object NewItem {
+
+  case class Meta(
+      language: Option[Language],
+      direction: Option[Direction],
+      folderId: Option[Ident],
+      source: Option[String],
+      skipDuplicate: Option[Boolean],
+      tags: Option[List[String]],
+      attachmentsOnly: Option[Boolean]
+  ) {
+
+    def toProcessArgs(
+        cid: Ident,
+        collLang: Option[Language],
+        sourceAbbrev: String
+    ): ProcessItemArgs.ProcessMeta =
+      ProcessItemArgs.ProcessMeta(
+        collective = cid,
+        itemId = None,
+        language = language.orElse(collLang).getOrElse(Language.English),
+        direction = direction,
+        sourceAbbrev = source.getOrElse(sourceAbbrev),
+        folderId = folderId,
+        validFileTypes = Seq.empty,
+        skipDuplicate = skipDuplicate.getOrElse(true),
+        fileFilter = None,
+        tags = tags,
+        reprocess = false,
+        attachmentsOnly = attachmentsOnly
+      )
+  }
+
+  object Meta {
+    implicit val jsonEncoder: Encoder[Meta] = deriveEncoder
+    implicit val jsonDecoder: Decoder[Meta] = deriveDecoder
+  }
+
+  implicit val jsonDecoder: Decoder[NewItem] = deriveDecoder
+  implicit val jsonEncoder: Encoder[NewItem] = deriveEncoder
+}

modules/addonlib/src/main/scala/docspell/addons/package.scala

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell
+
+import cats.data.Kleisli
+
+package object addons {
+
+  type AddonExec[F[_]] = Kleisli[F, InputEnv, AddonExecutionResult]
+
+}

modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.Applicative
+import cats.effect.{Ref, Sync}
+import cats.syntax.all._
+import fs2.Pipe
+
+trait CollectOut[F[_]] {
+
+  def get: F[String]
+
+  def append: Pipe[F, String, String]
+}
+
+object CollectOut {
+
+  def none[F[_]: Applicative]: CollectOut[F] =
+    new CollectOut[F] {
+      def get = "".pure[F]
+      def append = identity
+    }
+
+  def buffer[F[_]: Sync]: F[CollectOut[F]] =
+    Ref
+      .of[F, Vector[String]](Vector.empty)
+      .map(buffer =>
+        new CollectOut[F] {
+          override def get =
+            buffer.get.map(_.mkString("\n").trim)
+
+          override def append =
+            _.evalTap(line =>
+              if (line.trim.nonEmpty) buffer.update(_.appended(line)) else ().pure[F]
+            )
+        }
+      )
+}

modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import fs2.io.file.Path
+
+import docspell.common.Duration
+import docspell.common.exec.{Args, Env, SysCmd}
+
+/** Builder for a docker system command. */
+case class DockerBuilder(
+    dockerBinary: String,
+    subCmd: String,
+    timeout: Duration,
+    containerName: Option[String] = None,
+    env: Env = Env.empty,
+    mounts: Args = Args.empty,
+    network: Option[String] = Some("host"),
+    workingDir: Option[String] = None,
+    imageName: Option[String] = None,
+    cntCmd: Args = Args.empty
+) {
+  def containerCmd(args: Args): DockerBuilder =
+    copy(cntCmd = args)
+  def containerCmd(args: Seq[String]): DockerBuilder =
+    copy(cntCmd = Args(args))
+
+  def imageName(name: String): DockerBuilder =
+    copy(imageName = Some(name))
+
+  def workDirectory(dir: String): DockerBuilder =
+    copy(workingDir = Some(dir))
+
+  def withDockerBinary(bin: String): DockerBuilder =
+    copy(dockerBinary = bin)
+
+  def withSubCmd(cmd: String): DockerBuilder =
+    copy(subCmd = cmd)
+
+  def withEnv(key: String, value: String): DockerBuilder =
+    copy(env = env.add(key, value))
+
+  def withEnv(moreEnv: Env): DockerBuilder =
+    copy(env = env ++ moreEnv)
+
+  def privateNetwork(flag: Boolean): DockerBuilder =
+    if (flag) copy(network = Some("none"))
+    else copy(network = Some("host"))
+
+  def mount(
+      hostDir: Path,
+      cntDir: Option[String] = None,
+      readOnly: Boolean = true
+  ): DockerBuilder = {
+    val target = cntDir.getOrElse(hostDir.toString)
+    val ro = Option.when(readOnly)(",readonly").getOrElse("")
+    val opt = s"type=bind,source=$hostDir,target=$target${ro}"
+    copy(mounts = mounts.append("--mount", opt))
+  }
+
+  def withName(containerName: String): DockerBuilder =
+    copy(containerName = Some(containerName))
+
+  def build: SysCmd =
+    SysCmd(dockerBinary, buildArgs).withTimeout(timeout)
+
+  private def buildArgs: Args =
+    Args
+      .of(subCmd)
+      .append("--rm")
+      .option("--name", containerName)
+      .append(mounts)
+      .option("--network", network)
+      .append(env.mapConcat((k, v) => List("--env", s"${k}=${v}")))
+      .option("-w", workingDir)
+      .appendOpt(imageName)
+      .append(cntCmd)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.addons.AddonExecutorConfig.DockerConfig
+import docspell.addons._
+import docspell.common.Duration
+import docspell.common.exec.{Env, SysCmd, SysExec}
+import docspell.common.util.Random
+import docspell.logging.Logger
+
+final class DockerRunner[F[_]: Async](cfg: DockerRunner.Config) extends AddonRunner[F] {
+
+  val runnerType = List(RunnerType.Docker)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ) = for {
+    _ <- OptionT.whenF(requireBuild(ctx))(build(logger, ctx)).value
+    suffix <- Random[F].string(4)
+    cmd = createDockerCommand(env, ctx, suffix)
+    result <- RunnerUtil.runAddonCommand(logger, cmd, ctx)
+  } yield result
+
+  def createDockerCommand(
+      env: Env,
+      ctx: Context,
+      suffix: String
+  ): SysCmd = {
+    val outputPath = "/mnt/output"
+    val cachePath = "/mnt/cache"
+    val addonArgs =
+      ctx.addonCommand(
+        "",
+        Duration.zero,
+        relativeToBase = true,
+        outputPath.some,
+        cachePath.some
+      )
+
+    DockerBuilder(cfg.docker.dockerBinary, "run", cfg.timeout)
+      .withName(ctx.meta.nameAndVersion + "-" + suffix)
+      .withEnv(env)
+      .withEnv(addonArgs.env)
+      .mount(ctx.baseDir, "/mnt/work".some, readOnly = false)
+      .mount(ctx.outputDir, outputPath.some, readOnly = false)
+      .mount(ctx.cacheDir, cachePath.some, readOnly = false)
+      .workDirectory("/mnt/work")
+      .privateNetwork(ctx.meta.isPure)
+      .imageName(imageName(ctx))
+      .containerCmd(addonArgs.args)
+      .build
+  }
+
+  def build(logger: Logger[F], ctx: Context): F[Unit] =
+    for {
+      _ <- logger.info(s"Building docker image for addon ${ctx.meta.nameAndVersion}")
+      cmd = cfg.docker.dockerBuild(imageName(ctx))
+      _ <- SysExec(cmd, logger, ctx.addonDir.some)
+        .flatMap(_.logOutputs(logger, "docker build"))
+        .use(_.waitFor())
+      _ <- logger.info(s"Docker image built successfully")
+    } yield ()
+
+  private def requireBuild(ctx: Context) =
+    ctx.meta.runner
+      .flatMap(_.docker)
+      .flatMap(_.image)
+      .isEmpty
+
+  private def imageName(ctx: Context): String =
+    ctx.meta.runner
+      .flatMap(_.docker)
+      .flatMap(_.image)
+      .getOrElse(s"${ctx.meta.meta.name}:latest")
+}
+
+object DockerRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): DockerRunner[F] =
+    new DockerRunner[F](Config(cfg.dockerRunner, cfg.runTimeout))
+
+  case class Config(docker: DockerConfig, timeout: Duration)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import fs2.io.file.Path
+
+import docspell.common.exec.{Args, Env, SysCmd}
+
+case class NSpawnBuilder(
+    child: SysCmd,
+    chroot: Path,
+    spawnBinary: String = "systemd-nspawn",
+    sudoBinary: String = "sudo",
+    args: Args = Args.empty,
+    env: Env = Env.empty
+) {
+
+  def withNSpawnBinary(bin: String): NSpawnBuilder =
+    copy(spawnBinary = bin)
+
+  def withSudoBinary(bin: String): NSpawnBuilder =
+    copy(sudoBinary = bin)
+
+  def withEnv(key: String, value: String): NSpawnBuilder =
+    copy(args = args.append(s"--setenv=$key=$value"))
+
+  def withEnvOpt(key: String, value: Option[String]): NSpawnBuilder =
+    value.map(v => withEnv(key, v)).getOrElse(this)
+
+  def withName(containerName: String): NSpawnBuilder =
+    copy(args = args.append(s"--machine=$containerName"))
+
+  def mount(
+      hostDir: Path,
+      cntDir: Option[String] = None,
+      readOnly: Boolean = true
+  ): NSpawnBuilder = {
+    val bind = if (readOnly) "--bind-ro" else "--bind"
+    val target = cntDir.map(dir => s":$dir").getOrElse("")
+    copy(args = args.append(s"${bind}=${hostDir}${target}"))
+  }
+
+  def workDirectory(dir: String): NSpawnBuilder =
+    copy(args = args.append(s"--chdir=$dir"))
+
+  def portMap(port: Int): NSpawnBuilder =
+    copy(args = args.append("-p", port.toString))
+
+  def privateNetwork(flag: Boolean): NSpawnBuilder =
+    if (flag) copy(args = args.append("--private-network"))
+    else this
+
+  def build: SysCmd =
+    SysCmd(
+      program = if (sudoBinary.nonEmpty) sudoBinary else spawnBinary,
+      args = buildArgs,
+      timeout = child.timeout,
+      env = env
+    )
+
+  private def buildArgs: Args =
+    Args
+      .of("--private-users=identity") // can't use -U because need writeable bind mounts
+      .append("--notify-ready=yes")
+      .append("--ephemeral")
+      .append("--as-pid2")
+      .append("--console=pipe")
+      .append("--no-pager")
+      .append("--bind-ro=/bin")
+      .append("--bind-ro=/usr/bin")
+      .append("--bind-ro=/nix/store")
+      .append(s"--directory=$chroot")
+      .append(args)
+      .append(child.env.map((n, v) => s"--setenv=$n=$v"))
+      .prependWhen(sudoBinary.nonEmpty)(spawnBinary)
+      .append("--")
+      .append(child.program)
+      .append(child.args)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.addons.AddonExecutorConfig.{NSpawn, NixConfig}
+import docspell.addons._
+import docspell.addons.runner.NixFlakeRunner.PreCtx
+import docspell.common.Duration
+import docspell.common.exec._
+import docspell.logging.Logger
+
+final class NixFlakeRunner[F[_]: Async](cfg: NixFlakeRunner.Config)
+    extends AddonRunner[F] {
+
+  val runnerType = List(RunnerType.NixFlake)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ): F[AddonResult] =
+    prepare(logger, ctx)
+      .flatMap { preCtx =>
+        if (preCtx.nspawnEnabled) runInContainer(logger, env, preCtx, ctx)
+        else runOnHost(logger, env, preCtx, ctx)
+      }
+
+  def prepare(logger: Logger[F], ctx: Context): F[PreCtx] =
+    for {
+      _ <- logger.info(s"Prepare addon ${ctx.meta.nameAndVersion} for executing via nix")
+      _ <- logger.debug(s"Building with nix build")
+      _ <- SysExec(cfg.nixBuild, logger, workdir = ctx.addonDir.some)
+        .flatMap(_.logOutputs(logger, "nix build"))
+        .use(_.waitFor())
+      bin <- findFile(ctx.addonDir / "result" / "bin", ctx.addonDir / "result")
+      _ <- logger.debug(s"Build done, found binary: $bin")
+      _ <- logger.debug(s"Checking for systemd-nspawn…")
+      cnt <- checkContainer(logger)
+      _ <-
+        if (cnt)
+          logger.debug(s"Using systemd-nspawn to run addon in a container.")
+        else
+          logger.info(s"Running via systemd-nspawn is disabled in the config file")
+    } yield PreCtx(cnt, ctx.baseDir.relativize(bin))
+
+  private def checkContainer(logger: Logger[F]): F[Boolean] =
+    if (!cfg.nspawn.enabled) false.pure[F]
+    else RunnerUtil.checkContainer(logger, cfg.nspawn)
+
+  private def runOnHost(
+      logger: Logger[F],
+      env: Env,
+      preCtx: PreCtx,
+      ctx: Context
+  ): F[AddonResult] = {
+    val cmd =
+      SysCmd(preCtx.binary.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env)
+    RunnerUtil.runDirectly(logger, ctx)(cmd)
+  }
+
+  private def runInContainer(
+      logger: Logger[F],
+      env: Env,
+      preCtx: PreCtx,
+      ctx: Context
+  ): F[AddonResult] = {
+    val cmd = SysCmd(preCtx.binary.toString, Args.empty)
+      .withTimeout(cfg.timeout)
+      .addEnv(env)
+    RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd)
+  }
+
+  /** Find first file, try directories in given order. */
+  private def findFile(firstDir: Path, more: Path*): F[Path] = {
+    val fail: F[Path] = Sync[F].raiseError(
+      new NoSuchElementException(
+        s"No file found to execute in ${firstDir :: more.toList}"
+      )
+    )
+
+    Stream
+      .emits(more)
+      .cons1(firstDir)
+      .flatMap(dir =>
+        Files[F]
+          .list(dir)
+          .evalFilter(p => Files[F].isDirectory(p).map(!_))
+          .take(1)
+      )
+      .take(1)
+      .compile
+      .last
+      .flatMap(_.fold(fail)(Sync[F].pure))
+  }
+}
+
+object NixFlakeRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): NixFlakeRunner[F] =
+    new NixFlakeRunner[F](Config(cfg.nixRunner, cfg.nspawn, cfg.runTimeout))
+
+  case class Config(
+      nix: NixConfig,
+      nspawn: NSpawn,
+      timeout: Duration
+  ) {
+
+    val nixBuild =
+      SysCmd(nix.nixBinary, Args.of("build")).withTimeout(nix.buildTimeout)
+
+    val nspawnVersion = nspawn.nspawnVersion
+  }
+
+  case class PreCtx(nspawnEnabled: Boolean, binary: Path)
+}

modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect.{Async, Sync}
+import cats.syntax.all._
+import fs2.Pipe
+import fs2.io.file.Files
+
+import docspell.addons._
+import docspell.addons.out.AddonOutput
+import docspell.common.exec.{SysCmd, SysExec}
+import docspell.common.util.Random
+import docspell.logging.Logger
+
+import io.circe.{parser => JsonParser}
+
+private[addons] object RunnerUtil {
+
+  /** Run the given `cmd` on this machine.
+    *
+    * The `cmd` is containing a template command to execute the addon. The path are
+    * expected to be relative to the `ctx.baseDir`. Additional arguments and environment
+    * variables are added as configured in the addon.
+    */
+  def runDirectly[F[_]: Async](
+      logger: Logger[F],
+      ctx: Context
+  )(cmd: SysCmd): F[AddonResult] = {
+    val addonCmd = ctx
+      .addonCommand(cmd.program, cmd.timeout, relativeToBase = false, None, None)
+      .withArgs(_.append(cmd.args))
+      .addEnv(cmd.env)
+    runAddonCommand(logger, addonCmd, ctx)
+  }
+
+  /** Run the given `cmd` inside a container via systemd-nspawn.
+    *
+    * The `cmd` is containing a template command to execute the addon. The path are
+    * expected to be relative to the `ctx.baseDir`. Additional arguments and environment
+    * variables are added as configured in the addon.
+    */
+  def runInContainer[F[_]: Async](
+      logger: Logger[F],
+      cfg: AddonExecutorConfig.NSpawn,
+      ctx: Context
+  )(cmd: SysCmd): F[AddonResult] = {
+    val outputPath = "/mnt/output"
+    val cachePath = "/mnt/cache"
+    val addonCmd = ctx
+      .addonCommand(
+        cmd.program,
+        cmd.timeout,
+        relativeToBase = true,
+        outputPath.some,
+        cachePath.some
+      )
+      .withArgs(_.append(cmd.args))
+      .addEnv(cmd.env)
+
+    val chroot = ctx.baseDir / "cnt-root"
+    val nspawn = NSpawnBuilder(addonCmd, chroot)
+      .withNSpawnBinary(cfg.nspawnBinary)
+      .withSudoBinary(cfg.sudoBinary)
+      .mount(ctx.baseDir, "/mnt/work".some, readOnly = false)
+      .mount(ctx.cacheDir, cachePath.some, readOnly = false)
+      .mount(ctx.outputDir, outputPath.some, readOnly = false)
+      .workDirectory("/mnt/work")
+      .withEnv("XDG_RUNTIME_DIR", "/mnt/work")
+      .privateNetwork(ctx.meta.isPure)
+
+    for {
+      suffix <- Random[F].string(4)
+      _ <- List(chroot).traverse_(Files[F].createDirectories)
+      res <- runAddonCommand(
+        logger,
+        nspawn.withName(ctx.meta.nameAndVersion + "-" + suffix).build,
+        ctx
+      )
+      // allow some time to unregister the current container
+      // only important when same addons are called in sequence too fast
+      _ <- Sync[F].sleep(cfg.containerWait.toScala)
+    } yield res
+  }
+
+  private def procPipe[F[_]](
+      p: String,
+      ctx: Context,
+      collect: CollectOut[F],
+      logger: Logger[F]
+  ): Pipe[F, String, Unit] =
+    _.through(collect.append)
+      .map(line => s">> [${ctx.meta.nameAndVersion} ($p)] $line")
+      .evalMap(logger.debug(_))
+
+  /** Runs the external command that is executing the addon.
+    *
+    * If the addons specifies to collect its output, the stdout is parsed as json and
+    * decoded into [[AddonOutput]].
+    */
+  def runAddonCommand[F[_]: Async](
+      logger: Logger[F],
+      cmd: SysCmd,
+      ctx: Context
+  ): F[AddonResult] =
+    for {
+      stdout <-
+        if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F]
+        else CollectOut.none[F].pure[F]
+      cmdResult <- SysExec(cmd, logger, ctx.baseDir.some)
+        .flatMap(
+          _.consumeOutputs(
+            procPipe("out", ctx, stdout, logger),
+            procPipe("err", ctx, CollectOut.none[F], logger)
+          )
+        )
+        .use(_.waitFor())
+        .attempt
+      addonResult <- cmdResult match {
+        case Right(rc) if rc != 0 =>
+          for {
+            _ <- logger.error(
+              s"Addon ${ctx.meta.nameAndVersion} returned non-zero: $rc"
+            )
+          } yield AddonResult.executionError(rc)
+
+        case Right(_) =>
+          for {
+            _ <- logger.debug(s"Addon ${ctx.meta.nameAndVersion} executed successfully!")
+            out <- stdout.get
+            _ <- logger.debug(s"Addon stdout: $out")
+            result = Option
+              .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)(
+                JsonParser
+                  .decode[AddonOutput](out)
+                  .fold(AddonResult.decodingError, AddonResult.success)
+              )
+              .getOrElse(AddonResult.empty)
+          } yield result
+
+        case Left(ex) =>
+          logger
+            .error(ex)(s"Executing external command failed!")
+            .as(AddonResult.executionFailed(ex))
+      }
+    } yield addonResult
+
+  /** Check whether `systemd-nspawn` is available on this machine. */
+  def checkContainer[F[_]: Async](
+      logger: Logger[F],
+      cfg: AddonExecutorConfig.NSpawn
+  ): F[Boolean] =
+    for {
+      rc <- SysExec(cfg.nspawnVersion, logger)
+        .flatMap(_.logOutputs(logger, "nspawn"))
+        .use(_.waitFor())
+      _ <-
+        OptionT
+          .whenF(rc != 0)(
+            logger.warn(
+              s"No systemd-nspawn found! Addon is not executed inside a container."
+            )
+          )
+          .value
+    } yield rc == 0
+}

modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons.runner
+
+import cats.data.OptionT
+import cats.effect._
+import cats.kernel.Monoid
+import cats.syntax.all._
+import fs2.io.file.PosixPermission._
+import fs2.io.file.{Files, PosixPermissions}
+
+import docspell.addons.AddonExecutorConfig.NSpawn
+import docspell.addons._
+import docspell.common.Duration
+import docspell.common.exec.{Args, Env, SysCmd}
+import docspell.logging.Logger
+
+final class TrivialRunner[F[_]: Async](cfg: TrivialRunner.Config) extends AddonRunner[F] {
+  private val sync = Async[F]
+  private val files = Files[F]
+  implicit val andMonoid: Monoid[Boolean] = Monoid.instance[Boolean](true, _ && _)
+
+  private val executeBits = PosixPermissions(
+    OwnerExecute,
+    OwnerRead,
+    OwnerWrite,
+    GroupExecute,
+    GroupRead,
+    OthersExecute,
+    OthersRead
+  )
+
+  val runnerType = List(RunnerType.Trivial)
+
+  def run(
+      logger: Logger[F],
+      env: Env,
+      ctx: Context
+  ) = {
+    val binaryPath = ctx.meta.runner
+      .flatMap(_.trivial)
+      .map(_.exec)
+      .map(bin => ctx.addonDir / bin)
+
+    binaryPath match {
+      case None =>
+        sync.raiseError(new IllegalStateException("No executable specified in addon!"))
+
+      case Some(file) =>
+        val bin = ctx.baseDir.relativize(file)
+        val cmd = SysCmd(bin.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env)
+
+        val withNSpawn =
+          OptionT
+            .whenF(cfg.nspawn.enabled)(RunnerUtil.checkContainer(logger, cfg.nspawn))
+            .getOrElse(false)
+
+        files.setPosixPermissions(file, executeBits).attempt *>
+          withNSpawn.flatMap {
+            case true =>
+              RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd)
+            case false =>
+              RunnerUtil.runDirectly(logger, ctx)(cmd)
+          }
+    }
+  }
+}
+
+object TrivialRunner {
+  def apply[F[_]: Async](cfg: AddonExecutorConfig): TrivialRunner[F] =
+    new TrivialRunner[F](Config(cfg.nspawn, cfg.runTimeout))
+
+  case class Config(nspawn: NSpawn, timeout: Duration)
+}

modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.option._
+
+import docspell.common.UrlReader
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  tempDir.test("Read archive from directory") { dir =>
+    for {
+      archive <- IO(AddonArchive(dummyAddonUrl, "", ""))
+      path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir)
+
+      aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some)
+      _ = {
+        assertEquals(aa.name, "dummy-addon")
+        assertEquals(aa.version, "2.9")
+        assertEquals(aa.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+
+  test("Read archive from zip file") {
+    for {
+      archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO])
+      _ = {
+        assertEquals(archive.name, "dummy-addon")
+        assertEquals(archive.version, "2.9")
+        assertEquals(archive.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.logging.{Level, TestLoggingConfig}
+
+import munit._
+
+class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig {
+  val logger = docspell.logging.getLogger[IO]
+
+  override def docspellLogConfig =
+    super.docspellLogConfig.copy(minimumLevel = Level.Trace)
+
+  tempDir.test("select docker if Dockerfile exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake if flake.nix exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.NixFlake))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake and docker") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake))
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.common.Glob
+import docspell.files.Zip
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  test("read meta from zip file") {
+    val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192))
+    assertIO(meta, dummyAddonMeta)
+  }
+
+  tempDir.test("read meta from directory") { dir =>
+    for {
+      _ <- dummyAddonUrl
+        .readURL[IO](8192)
+        .through(Zip.unzip(8192, Glob.all))
+        .through(Zip.saveTo(logger, dir, moveUp = true))
+        .compile
+        .drain
+      meta <- AddonMeta.findInDirectory[IO](dir)
+      _ = assertEquals(meta, dummyAddonMeta)
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.addons.out.AddonOutput
+
+import io.circe.parser.decode
+import munit.FunSuite
+
+class AddonOutputTest extends FunSuite {
+
+  test("decode empty object") {
+    val out = decode[AddonOutput]("{}")
+    println(out)
+  }
+
+  test("decode sample output") {
+    val jsonStr =
+      """{ "files": [
+        |    {
+        |      "itemId": "qZDnyGIAJsXr",
+        |      "textFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.txt"
+        |      },
+        |      "pdfFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.pdf"
+        |      }
+        |    }
+        |  ]
+        |}
+        |""".stripMargin
+
+    val out = decode[AddonOutput](jsonStr)
+    println(out)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import cats.effect.IO
+import fs2.io.file.Path
+
+import docspell.common.LenientUri
+import docspell.common.exec.Env
+import docspell.logging.{Logger, TestLoggingConfig}
+
+import munit._
+
+class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig {
+
+  val logger = docspell.logging.getLogger[IO]
+
+  val dummyContext = Context(
+    addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""),
+    meta = AddonMeta.empty("test", "1.0"),
+    baseDir = Path(""),
+    addonDir = Path(""),
+    outputDir = Path(""),
+    cacheDir = Path("")
+  )
+
+  test("firstSuccessful must stop on first success") {
+    val counter = new AtomicInteger(0)
+    val runner = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(runner, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("firstSuccessful must try with next on error") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.failWith[IO]("failed")
+    val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(fail, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("do not retry on decoding errors") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(fail, increment, increment)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 0)
+    } yield ()
+  }
+
+  test("try on errors but stop on decoding error") {
+    val counter = new AtomicInteger(0)
+    val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val incrementFail =
+      new MockRunner(IO(counter.incrementAndGet()).void)
+        .as(AddonResult.executionFailed(new Exception("fail")))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(
+      incrementFail,
+      incrementFail,
+      decodeFail,
+      increment,
+      increment
+    )
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 2)
+    } yield ()
+  }
+
+  final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty)
+      extends AddonRunner[IO] {
+    val runnerType = Nil
+    def run(
+        logger: Logger[IO],
+        env: Env,
+        ctx: Context
+    ) = run.as(result)
+
+    def as(r: AddonResult) = new MockRunner(run, r)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.{Files, Path, PosixPermissions}
+
+import docspell.addons.AddonExecutorConfig._
+import docspell.addons.AddonMeta._
+import docspell.addons.AddonTriggerType._
+import docspell.common.{Duration, LenientUri}
+import docspell.logging.TestLoggingConfig
+
+import munit.CatsEffectSuite
+
+trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
+
+  val files: Files[IO] = Files[IO]
+
+  val dummyAddonUrl =
+    LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip"))
+
+  val dummyAddonMeta =
+    AddonMeta(
+      meta =
+        AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some),
+      triggers = Some(
+        Set(Scheduled, FinalProcessItem, FinalReprocessItem)
+      ),
+      None,
+      runner = Runner(
+        nix = NixRunner(true).some,
+        docker = DockerRunner(
+          enable = true,
+          image = None,
+          build = "Dockerfile".some
+        ).some,
+        trivial = TrivialRunner(true, "src/addon.sh").some
+      ).some,
+      options = Options(networking = true, collectOutput = true).some
+    )
+
+  def baseTempDir: Path =
+    Path(s"/tmp/target/test-temp")
+
+  val tempDir =
+    ResourceFixture[Path](
+      Resource.eval(Files[IO].createDirectories(baseTempDir)) *>
+        Files[IO]
+          .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777"))
+    )
+
+  def testExecutorConfig(
+      runner: RunnerType,
+      runners: RunnerType*
+  ): AddonExecutorConfig = {
+    val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100))
+    AddonExecutorConfig(
+      runner :: runners.toList,
+      Duration.minutes(2),
+      nspawn,
+      NixConfig("nix", Duration.minutes(2)),
+      DockerConfig("docker", Duration.minutes(2))
+    )
+  }
+}

modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala

@@ -15,8 +15,8 @@ import fs2.io.file.{Files, Path}
 import docspell.analysis.classifier
 import docspell.analysis.classifier.TextClassifier._
 import docspell.analysis.nlp.Properties
-import docspell.common._
 import docspell.common.syntax.FileSyntax._
+import docspell.common.util.File
 import docspell.logging.Logger
 
 import edu.stanford.nlp.classify.ColumnDataClassifier

modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala

@@ -14,6 +14,7 @@ import cats.implicits._
 
 import docspell.analysis.NlpSettings
 import docspell.common._
+import docspell.common.util.File
 
 /** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and
   * initializing large objects.

modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala

@@ -17,6 +17,7 @@ import fs2.io.file.Files
 
 import docspell.analysis.classifier.TextClassifier.Data
 import docspell.common._
+import docspell.common.util.File
 import docspell.logging.TestLoggingConfig
 
 import munit._

modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala

@@ -13,6 +13,7 @@ import cats.effect.unsafe.implicits.global
 
 import docspell.analysis.Env
 import docspell.common._
+import docspell.common.util.File
 import docspell.files.TestFiles
 import docspell.logging.TestLoggingConfig
 

modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala

@@ -20,6 +20,7 @@ trait AttachedEvent[R] {
 
 object AttachedEvent {
 
+  /** Only the result, no events. */
   def only[R](v: R): AttachedEvent[R] =
     new AttachedEvent[R] {
       val value = v

modules/backend/src/main/scala/docspell/backend/BackendApp.scala

@@ -8,11 +8,14 @@ package docspell.backend
 
 import cats.effect._
 
+import docspell.backend.BackendCommands.EventContext
 import docspell.backend.auth.Login
 import docspell.backend.fulltext.CreateIndex
 import docspell.backend.ops._
 import docspell.backend.signup.OSignup
+import docspell.common.bc.BackendCommandRunner
 import docspell.ftsclient.FtsClient
+import docspell.joexapi.client.JoexClient
 import docspell.notification.api.{EventExchange, NotificationModule}
 import docspell.pubsub.api.PubSubT
 import docspell.scheduler.JobStoreModule
@@ -20,6 +23,7 @@ import docspell.store.Store
 import docspell.totp.Totp
 
 import emil.Emil
+import org.http4s.client.Client
 
 trait BackendApp[F[_]] {
 
@@ -35,6 +39,7 @@ trait BackendApp[F[_]] {
   def job: OJob[F]
   def item: OItem[F]
   def itemSearch: OItemSearch[F]
+  def attachment: OAttachment[F]
   def fulltext: OFulltext[F]
   def mail: OMail[F]
   def joex: OJoex[F]
@@ -52,23 +57,30 @@ trait BackendApp[F[_]] {
   def fileRepository: OFileRepository[F]
   def itemLink: OItemLink[F]
   def downloadAll: ODownloadAll[F]
+  def addons: OAddons[F]
+
+  def commands(eventContext: Option[EventContext]): BackendCommandRunner[F, Unit]
 }
 
 object BackendApp {
 
   def create[F[_]: Async](
+      cfg: Config,
       store: Store[F],
       javaEmil: Emil[F],
+      httpClient: Client[F],
       ftsClient: FtsClient[F],
       pubSubT: PubSubT[F],
       schedulerModule: JobStoreModule[F],
       notificationMod: NotificationModule[F]
   ): Resource[F, BackendApp[F]] =
     for {
+      nodeImpl <- ONode(store)
       totpImpl <- OTotp(store, Totp.default)
       loginImpl <- Login[F](store, Totp.default)
       signupImpl <- OSignup[F](store)
-      joexImpl <- OJoex(pubSubT)
+      joexClient = JoexClient(httpClient)
+      joexImpl <- OJoex(pubSubT, nodeImpl, joexClient)
       collImpl <- OCollective[F](
         store,
         schedulerModule.userTasks,
@@ -80,7 +92,6 @@ object BackendApp {
       equipImpl <- OEquipment[F](store)
       orgImpl <- OOrganization(store)
       uploadImpl <- OUpload(store, schedulerModule.jobs)
-      nodeImpl <- ONode(store)
       jobImpl <- OJob(store, joexImpl, pubSubT)
       createIndex <- CreateIndex.resource(ftsClient, store)
       itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs)
@@ -109,6 +120,16 @@ object BackendApp {
       fileRepoImpl <- OFileRepository(store, schedulerModule.jobs)
       itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl))
       downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs))
+      attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs))
+      addonsImpl <- Resource.pure(
+        OAddons(
+          cfg.addons,
+          store,
+          schedulerModule.userTasks,
+          schedulerModule.jobs,
+          joexImpl
+        )
+      )
     } yield new BackendApp[F] {
       val pubSub = pubSubT
       val login = loginImpl
@@ -139,5 +160,10 @@ object BackendApp {
       val fileRepository = fileRepoImpl
       val itemLink = itemLinkImpl
       val downloadAll = downloadAllImpl
+      val addons = addonsImpl
+      val attachment = attachImpl
+
+      def commands(eventContext: Option[EventContext]) =
+        BackendCommands.fromBackend(this, eventContext)
     }
 }

modules/backend/src/main/scala/docspell/backend/BackendCommands.scala

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend
+
+import cats.data.{NonEmptyList => Nel}
+import cats.effect.Sync
+import cats.syntax.all._
+
+import docspell.backend.BackendCommands.EventContext
+import docspell.backend.ops.OCustomFields.SetValue
+import docspell.backend.ops._
+import docspell.common.bc._
+import docspell.common.{AccountId, Ident, LenientUri}
+
+private[backend] class BackendCommands[F[_]: Sync](
+    itemOps: OItem[F],
+    attachOps: OAttachment[F],
+    fieldOps: OCustomFields[F],
+    notificationOps: ONotification[F],
+    eventContext: Option[EventContext]
+) extends BackendCommandRunner[F, Unit] {
+  private[this] val logger = docspell.logging.getLogger[F]
+
+  def run(collective: Ident, cmd: BackendCommand): F[Unit] =
+    doRun(collective, cmd).attempt.flatMap {
+      case Right(_) => ().pure[F]
+      case Left(ex) =>
+        logger.error(ex)(s"Backend command $cmd failed for collective ${collective.id}.")
+    }
+
+  def doRun(collective: Ident, cmd: BackendCommand): F[Unit] =
+    cmd match {
+      case BackendCommand.ItemUpdate(item, actions) =>
+        actions.traverse_(a => runItemAction(collective, item, a))
+
+      case BackendCommand.AttachmentUpdate(item, attach, actions) =>
+        actions.traverse_(a => runAttachAction(collective, item, attach, a))
+    }
+
+  def runAll(collective: Ident, cmds: List[BackendCommand]): F[Unit] =
+    cmds.traverse_(run(collective, _))
+
+  def runItemAction(collective: Ident, item: Ident, action: ItemAction): F[Unit] =
+    action match {
+      case ItemAction.AddTags(tags) =>
+        logger.debug(s"Setting tags $tags on ${item.id} for ${collective.id}") *>
+          itemOps
+            .linkTags(item, tags.toList, collective)
+            .flatMap(sendEvents)
+
+      case ItemAction.RemoveTags(tags) =>
+        logger.debug(s"Remove tags $tags on ${item.id} for ${collective.id}") *>
+          itemOps
+            .removeTagsMultipleItems(Nel.of(item), tags.toList, collective)
+            .flatMap(sendEvents)
+
+      case ItemAction.ReplaceTags(tags) =>
+        logger.debug(s"Replace tags $tags on ${item.id} for ${collective.id}") *>
+          itemOps
+            .setTags(item, tags.toList, collective)
+            .flatMap(sendEvents)
+
+      case ItemAction.SetFolder(folder) =>
+        logger.debug(s"Set folder $folder on ${item.id} for ${collective.id}") *>
+          itemOps
+            .setFolder(item, folder, collective)
+            .void
+
+      case ItemAction.RemoveTagsCategory(cats) =>
+        logger.debug(
+          s"Remove tags in categories $cats on ${item.id} for ${collective.id}"
+        ) *>
+          itemOps
+            .removeTagsOfCategories(item, collective, cats)
+            .flatMap(sendEvents)
+
+      case ItemAction.SetCorrOrg(id) =>
+        logger.debug(
+          s"Set correspondent organization ${id.map(_.id)} for ${collective.id}"
+        ) *>
+          itemOps.setCorrOrg(Nel.of(item), id, collective).void
+
+      case ItemAction.SetCorrPerson(id) =>
+        logger.debug(
+          s"Set correspondent person ${id.map(_.id)} for ${collective.id}"
+        ) *>
+          itemOps.setCorrPerson(Nel.of(item), id, collective).void
+
+      case ItemAction.SetConcPerson(id) =>
+        logger.debug(
+          s"Set concerning person ${id.map(_.id)} for ${collective.id}"
+        ) *>
+          itemOps.setConcPerson(Nel.of(item), id, collective).void
+
+      case ItemAction.SetConcEquipment(id) =>
+        logger.debug(
+          s"Set concerning equipment ${id.map(_.id)} for ${collective.id}"
+        ) *>
+          itemOps.setConcEquip(Nel.of(item), id, collective).void
+
+      case ItemAction.SetField(field, value) =>
+        logger.debug(
+          s"Set field on item ${item.id} ${field.id} to '$value' for ${collective.id}"
+        ) *>
+          fieldOps
+            .setValue(item, SetValue(field, value, collective))
+            .flatMap(sendEvents)
+
+      case ItemAction.SetNotes(notes) =>
+        logger.debug(s"Set notes on item ${item.id} for ${collective.id}") *>
+          itemOps.setNotes(item, notes, collective).void
+
+      case ItemAction.AddNotes(notes, sep) =>
+        logger.debug(s"Add notes on item ${item.id} for ${collective.id}") *>
+          itemOps.addNotes(item, notes, sep, collective).void
+
+      case ItemAction.SetName(name) =>
+        logger.debug(s"Set name '$name' on item ${item.id} for ${collective.id}") *>
+          itemOps.setName(item, name, collective).void
+    }
+
+  def runAttachAction(
+      collective: Ident,
+      itemId: Ident,
+      attachId: Ident,
+      action: AttachmentAction
+  ): F[Unit] =
+    action match {
+      case AttachmentAction.SetExtractedText(text) =>
+        attachOps.setExtractedText(
+          collective,
+          itemId,
+          attachId,
+          text.getOrElse("").pure[F]
+        )
+    }
+
+  private def sendEvents(result: AttachedEvent[_]): F[Unit] =
+    eventContext match {
+      case Some(ctx) =>
+        notificationOps.offerEvents(result.event(ctx.account, ctx.baseUrl))
+      case None => ().pure[F]
+    }
+}
+
+object BackendCommands {
+
+  /** If supplied, notification events will be send. */
+  case class EventContext(account: AccountId, baseUrl: Option[LenientUri])
+
+  def fromBackend[F[_]: Sync](
+      backendApp: BackendApp[F],
+      eventContext: Option[EventContext] = None
+  ): BackendCommandRunner[F, Unit] =
+    new BackendCommands[F](
+      backendApp.item,
+      backendApp.attachment,
+      backendApp.customFields,
+      backendApp.notification,
+      eventContext
+    )
+
+  def apply[F[_]: Sync](
+      item: OItem[F],
+      attachment: OAttachment[F],
+      fields: OCustomFields[F],
+      notification: ONotification[F],
+      eventContext: Option[EventContext] = None
+  ): BackendCommandRunner[F, Unit] =
+    new BackendCommands[F](item, attachment, fields, notification, eventContext)
+}

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -20,7 +20,8 @@ case class Config(
     mailDebug: Boolean,
     jdbc: JdbcConfig,
     signup: SignupConfig,
-    files: Config.Files
+    files: Config.Files,
+    addons: Config.Addons
 ) {
 
   def mailSettings: Settings =
@@ -66,4 +67,21 @@ object Config {
       (storesEmpty |+| defaultStorePresent).map(_ => this)
     }
   }
+
+  case class Addons(
+      enabled: Boolean,
+      allowImpure: Boolean,
+      allowedUrls: UrlMatcher,
+      deniedUrls: UrlMatcher
+  ) {
+    def isAllowed(url: LenientUri): Boolean =
+      allowedUrls.matches(url) && !deniedUrls.matches(url)
+
+    def isDenied(url: LenientUri): Boolean =
+      !isAllowed(url)
+  }
+  object Addons {
+    val disabled: Addons =
+      Addons(false, false, UrlMatcher.False, UrlMatcher.True)
+  }
 }

modules/backend/src/main/scala/docspell/backend/JobFactory.scala

@@ -16,6 +16,26 @@ import docspell.notification.api.PeriodicQueryArgs
 import docspell.scheduler.Job
 
 object JobFactory extends MailAddressCodec {
+  def existingItemAddon[F[_]: Sync](
+      args: ItemAddonTaskArgs,
+      submitter: AccountId
+  ): F[Job[ItemAddonTaskArgs]] =
+    Job.createNew(
+      ItemAddonTaskArgs.taskName,
+      submitter.collective,
+      args,
+      "Run addons on item",
+      submitter.user,
+      Priority.High,
+      args.addonRunConfigs
+        .map(_.take(23))
+        .toList
+        .sorted
+        .foldLeft(args.itemId)(_ / _)
+        .take(250)
+        .some
+    )
+
   def downloadZip[F[_]: Sync](
       args: DownloadZipArgs,
       summaryId: Ident,

modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala

@@ -45,7 +45,14 @@ object CreateIndex {
           chunkSize: Int
       ): F[Unit] = {
         val attachs = store
-          .transact(QAttachment.allAttachmentMetaAndName(collective, itemIds, chunkSize))
+          .transact(
+            QAttachment.allAttachmentMetaAndName(
+              collective,
+              itemIds,
+              ItemState.validStates,
+              chunkSize
+            )
+          )
           .map(caa =>
             TextData
               .attachment(

modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.joex
+
+import fs2.io.file.Path
+
+import docspell.addons.AddonExecutorConfig
+
+final case class AddonEnvConfig(
+    workingDir: Path,
+    cacheDir: Path,
+    executorConfig: AddonExecutorConfig
+)

modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala

@@ -0,0 +1,199 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.joex
+
+import cats.data.OptionT
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.addons._
+import docspell.backend.joex.AddonOps.{AddonRunConfigRef, ExecResult}
+import docspell.backend.ops.OAttachment
+import docspell.common._
+import docspell.common.bc.BackendCommandRunner
+import docspell.common.exec.Env
+import docspell.logging.Logger
+import docspell.scheduler.JobStore
+import docspell.store.Store
+import docspell.store.file.FileUrlReader
+import docspell.store.records.AddonRunConfigResolved
+
+trait AddonOps[F[_]] {
+
+  def execAll(
+      collective: Ident,
+      trigger: Set[AddonTriggerType],
+      runConfigIds: Set[Ident],
+      logger: Option[Logger[F]]
+  )(
+      middleware: Middleware[F]
+  ): F[ExecResult]
+
+  def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])(
+      middleware: Middleware[F]
+  ): F[ExecResult]
+
+  /** Find enabled addon run config references to be executed. Can be additionally
+    * filtered by given ids and triggers.
+    */
+  def findAddonRefs(
+      collective: Ident,
+      trigger: Set[AddonTriggerType],
+      runConfigIds: Set[Ident]
+  ): F[List[AddonRunConfigRef]]
+
+  /** Find enabled addon run config reference given an addon task id */
+  def findAddonRef(collective: Ident, runConfigId: Ident): F[Option[AddonRunConfigRef]]
+
+  /** Creates an executor for addons given a configuration. */
+  def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]]
+
+}
+
+object AddonOps {
+  case class AddonRunConfigRef(
+      id: Ident,
+      collective: Ident,
+      userId: Option[Ident],
+      name: String,
+      refs: List[AddonRef]
+  )
+
+  object AddonRunConfigRef {
+    def fromResolved(r: AddonRunConfigResolved): AddonRunConfigRef =
+      AddonRunConfigRef(
+        r.config.id,
+        r.config.cid,
+        r.config.userId,
+        r.config.name,
+        r.refs.map(ref => AddonRef(ref.archive.asArchive, ref.ref.args))
+      )
+  }
+
+  case class ExecResult(
+      result: List[AddonExecutionResult],
+      runConfigs: List[AddonRunConfigRef]
+  ) {
+    lazy val combined = result.combineAll
+  }
+
+  object ExecResult {
+    def runConfigNotFound(id: Ident): ExecResult =
+      ExecResult(
+        AddonExecutionResult(
+          AddonResult.executionFailed(
+            new Exception(s"Addon run config ${id.id} not found.")
+          ) :: Nil,
+          false
+        ) :: Nil,
+        Nil
+      )
+  }
+
+  def apply[F[_]: Async](
+      cfg: AddonEnvConfig,
+      store: Store[F],
+      cmdRunner: BackendCommandRunner[F, Unit],
+      attachment: OAttachment[F],
+      jobStore: JobStore[F]
+  ): AddonOps[F] =
+    new AddonOps[F] with LoggerExtension {
+      private[this] val logger = docspell.logging.getLogger[F]
+
+      private val urlReader = FileUrlReader(store.fileRepo)
+      private val postProcess = AddonPostProcess(cmdRunner, store, attachment, jobStore)
+      private val prepare = new AddonPrepare[F](store)
+
+      def execAll(
+          collective: Ident,
+          trigger: Set[AddonTriggerType],
+          runConfigIds: Set[Ident],
+          logger: Option[Logger[F]]
+      )(
+          custom: Middleware[F]
+      ): F[ExecResult] =
+        for {
+          runCfgs <- findAddonRefs(collective, trigger, runConfigIds)
+          log = logger.getOrElse(this.logger)
+          _ <- log.info(s"Running ${runCfgs.size} addon tasks for trigger $trigger")
+
+          results <- runCfgs.traverse(r => execRunConfig(log, r, custom))
+        } yield ExecResult(results.flatMap(_.result), runCfgs)
+
+      def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])(
+          custom: Middleware[F]
+      ): F[ExecResult] =
+        (for {
+          runCfg <- OptionT(findAddonRef(collective, runConfigId))
+          execRes <- OptionT.liftF(execRunConfig(logger, runCfg, custom))
+        } yield execRes).getOrElse(ExecResult.runConfigNotFound(runConfigId))
+
+      def execRunConfig(
+          logger: Logger[F],
+          runCfg: AddonRunConfigRef,
+          custom: Middleware[F]
+      ): F[ExecResult] =
+        for {
+          executor <- getExecutor(cfg.executorConfig)
+          log = logger.withRunConfig(runCfg)
+          result <-
+            Directory.temp(cfg.workingDir, "addon-output-").use { outDir =>
+              val cacheDir = cfg.cacheDir / runCfg.id.id
+              val inputEnv =
+                InputEnv(runCfg.refs, cfg.workingDir, outDir, cacheDir, Env.empty)
+
+              for {
+                middleware <- createMiddleware(custom, runCfg)
+                res <- middleware(executor.execute(log)).run(inputEnv)
+                _ <- log.debug(s"Addon result: $res")
+                _ <- postProcess.onResult(log, runCfg.collective, res, outDir)
+              } yield res
+            }
+          execRes = ExecResult(List(result), List(runCfg))
+        } yield execRes
+
+      def createMiddleware(custom: Middleware[F], runCfg: AddonRunConfigRef) = for {
+        dscMW <- prepare.createDscEnv(runCfg, cfg.executorConfig.runTimeout)
+        mm = dscMW >> custom >> prepare.logResult(logger, runCfg) >> Middleware
+          .ephemeralRun[F]
+      } yield mm
+
+      def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] =
+        Async[F].pure(AddonExecutor(cfg, urlReader))
+
+      def findAddonRefs(
+          collective: Ident,
+          trigger: Set[AddonTriggerType],
+          runConfigIds: Set[Ident]
+      ): F[List[AddonRunConfigRef]] =
+        store
+          .transact(
+            AddonRunConfigResolved.findAllForCollective(
+              collective,
+              enabled = true.some,
+              trigger,
+              runConfigIds
+            )
+          )
+          .map(_.map(AddonRunConfigRef.fromResolved))
+
+      def findAddonRef(
+          collective: Ident,
+          runConfigId: Ident
+      ): F[Option[AddonRunConfigRef]] =
+        OptionT(
+          store
+            .transact(
+              AddonRunConfigResolved.findById(
+                runConfigId,
+                collective,
+                enabled = Some(true)
+              )
+            )
+        ).map(AddonRunConfigRef.fromResolved).value
+    }
+}

modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala

@@ -0,0 +1,198 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.joex
+
+import cats.data.OptionT
+import cats.effect.kernel.Sync
+import cats.syntax.all._
+import fs2.io.file.{Files, Path}
+
+import docspell.addons._
+import docspell.addons.out.{AddonOutput, ItemFile, NewItem}
+import docspell.backend.JobFactory
+import docspell.backend.ops.OAttachment
+import docspell.common._
+import docspell.common.bc.BackendCommandRunner
+import docspell.files.FileSupport
+import docspell.logging.Logger
+import docspell.scheduler.JobStore
+import docspell.store.Store
+import docspell.store.records._
+
+final private[joex] class AddonPostProcess[F[_]: Sync: Files](
+    cmdRunner: BackendCommandRunner[F, Unit],
+    store: Store[F],
+    attachOps: OAttachment[F],
+    jobStore: JobStore[F]
+) extends FileSupport {
+
+  def onResult(
+      logger: Logger[F],
+      collective: Ident,
+      result: AddonExecutionResult,
+      outputDir: Path
+  ): F[Unit] =
+    result.addonResult match {
+      case AddonResult.Success(output) =>
+        onSuccess(logger, collective, output, outputDir)
+      case _ =>
+        ().pure[F]
+    }
+
+  def onSuccess(
+      logger: Logger[F],
+      collective: Ident,
+      output: AddonOutput,
+      outputDir: Path
+  ): F[Unit] =
+    for {
+      _ <- logger.info("Applying addon output")
+      _ <- cmdRunner.runAll(collective, output.commands)
+      _ <- logger.debug("Applying changes from files")
+      _ <- output.files.traverse_(updateOne(logger, collective, outputDir))
+      _ <- output.newItems.traverse_(submitNewItem(logger, collective, outputDir))
+    } yield ()
+
+  def submitNewItem(
+      logger: Logger[F],
+      collective: Ident,
+      outputDir: Path
+  )(newItem: NewItem): F[Unit] =
+    for {
+      _ <- logger.info(s"Submit new item with ${newItem.files.size} files")
+      files <- newItem.resolveFiles[F](logger, outputDir)
+      collLang <- store.transact(RCollective.findLanguage(collective))
+      uploaded <- files.traverse(file =>
+        file.readAll
+          .through(
+            store.fileRepo.save(
+              collective,
+              FileCategory.AttachmentSource,
+              MimeTypeHint.filename(file)
+            )
+          )
+          .compile
+          .lastOrError
+          .map(key => file.fileName.toString -> key)
+      )
+      _ <- logger.debug(s"Saved ${uploaded.size} files to be processed.")
+      args = ProcessItemArgs(
+        newItem.toProcessMeta(collective, collLang, "addon"),
+        uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2))
+      )
+      account = AccountId(collective, DocspellSystem.user)
+      job <- JobFactory.processItem(args, account, Priority.High, None)
+      _ <- jobStore.insert(job.encode)
+      _ <- logger.debug(s"Submitted job for processing: ${job.id}")
+    } yield ()
+
+  def updateOne(logger: Logger[F], collective: Ident, outputDir: Path)(
+      itemFile: ItemFile
+  ): F[Unit] =
+    for {
+      textFiles <- itemFile.resolveTextFiles(logger, outputDir)
+      pdfFiles <- itemFile.resolvePdfFiles(logger, outputDir)
+      previewFiles <- itemFile.resolvePreviewFiles(logger, outputDir)
+      attachs <- OptionT
+        .whenF(textFiles.nonEmpty || pdfFiles.nonEmpty || previewFiles.nonEmpty)(
+          store.transact(RAttachment.findByItem(itemFile.itemId))
+        )
+        .getOrElse(Vector.empty)
+      _ <- textFiles.traverse_ { case (key, file) =>
+        withAttach(logger, key, attachs) { ra =>
+          setText(collective, ra, file.readText)
+        }
+      }
+      _ <- pdfFiles.traverse_ { case (key, file) =>
+        withAttach(logger, key, attachs) { ra =>
+          replacePdf(collective, ra, file, previewFiles.forall(_._1 != key))
+        }
+      }
+      _ <- previewFiles.traverse_ { case (key, file) =>
+        withAttach(logger, key, attachs) { ra =>
+          replacePreview(collective, ra.id, file)
+        }
+      }
+      _ <- submitNewFiles(logger, collective, outputDir)(itemFile)
+    } yield ()
+
+  def submitNewFiles(
+      logger: Logger[F],
+      collective: Ident,
+      outputDir: Path
+  )(itemFile: ItemFile): F[Unit] =
+    for {
+      _ <- logger.info(s"Submitting new file for item")
+      collLang <- store.transact(RCollective.findLanguage(collective))
+      newFiles <- itemFile.resolveNewFiles(logger, outputDir)
+      byMeta = newFiles.groupBy(_._1.metadata).view.mapValues(_.map(_._2))
+      account = AccountId(collective, DocspellSystem.user)
+      _ <- byMeta.toList.traverse_ { case (meta, files) =>
+        for {
+          uploaded <- files.traverse(file =>
+            file.readAll
+              .through(
+                store.fileRepo.save(
+                  collective,
+                  FileCategory.AttachmentSource,
+                  MimeTypeHint.filename(file)
+                )
+              )
+              .compile
+              .lastOrError
+              .map(key => file.fileName.toString -> key)
+          )
+          args = ProcessItemArgs(
+            meta.toProcessMeta(collective, itemFile.itemId, collLang, "addon"),
+            uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2))
+          )
+          job <- JobFactory.processItem(args, account, Priority.High, None)
+          _ <- jobStore.insert(job.encode)
+          _ <- logger.debug(s"Submitted job for processing: ${job.id}")
+        } yield ()
+      }
+    } yield ()
+
+  private def withAttach(logger: Logger[F], key: String, attachs: Vector[RAttachment])(
+      run: RAttachment => F[Unit]
+  ): F[Unit] =
+    OptionT
+      .fromOption(
+        attachs.find(a => a.id.id == key || key.toIntOption == a.position.some)
+      )
+      .semiflatMap(run)
+      .getOrElseF(logger.warn(s"Cannot find attachment for $key to update text!"))
+
+  private def setText(collective: Ident, ra: RAttachment, readText: F[String]): F[Unit] =
+    attachOps.setExtractedText(collective, ra.itemId, ra.id, readText)
+
+  private def replacePdf(
+      collective: Ident,
+      ra: RAttachment,
+      file: Path,
+      generatePreview: Boolean
+  ): F[Unit] =
+    attachOps.addOrReplacePdf(collective, ra.id, file.readAll, generatePreview)
+
+  private def replacePreview(
+      collective: Ident,
+      attachId: Ident,
+      imageData: Path
+  ): F[Unit] =
+    attachOps.addOrReplacePreview(collective, attachId, imageData.readAll)
+}
+
+object AddonPostProcess {
+
+  def apply[F[_]: Sync: Files](
+      cmdRunner: BackendCommandRunner[F, Unit],
+      store: Store[F],
+      attachment: OAttachment[F],
+      jobStore: JobStore[F]
+  ): AddonPostProcess[F] =
+    new AddonPostProcess[F](cmdRunner, store, attachment, jobStore)
+}

modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.joex
+
+import cats.data.{Kleisli, OptionT}
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.addons.Middleware
+import docspell.backend.auth.AuthToken
+import docspell.backend.joex.AddonOps.AddonRunConfigRef
+import docspell.common._
+import docspell.logging.Logger
+import docspell.store.Store
+import docspell.store.records.{RNode, RUser}
+
+import scodec.bits.ByteVector
+
+private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExtension {
+
+  def logResult(logger: Logger[F], ref: AddonRunConfigRef): Middleware[F] =
+    Middleware(_.mapF(_.attempt.flatTap {
+      case Right(_) => ().pure[F]
+      case Left(ex) =>
+        logger
+          .withRunConfig(ref)
+          .warn(ex)(s"Addon task '${ref.id.id}' has failed")
+    }.rethrow))
+
+  /** Creates environment variables for dsc to connect to the docspell server for the
+    * given run config.
+    */
+  def createDscEnv(
+      runConfigRef: AddonRunConfigRef,
+      tokenValidity: Duration
+  ): F[Middleware[F]] =
+    (for {
+      userId <- OptionT.fromOption[F](runConfigRef.userId)
+      user <- OptionT(store.transact(RUser.getIdByIdOrLogin(userId)))
+      account = AccountId(runConfigRef.collective, user.login)
+      env =
+        Middleware.prepare[F](
+          Kleisli(input => makeDscEnv(account, tokenValidity).map(input.addEnv))
+        )
+    } yield env).getOrElse(Middleware.identity[F])
+
+  /** Creates environment variables to have dsc automatically connect as the given user.
+    * Additionally a random rest-server is looked up from the database to set its url.
+    */
+  def makeDscEnv(
+      accountId: AccountId,
+      tokenValidity: Duration
+  ): F[Map[String, String]] =
+    for {
+      serverNode <- store.transact(
+        RNode
+          .findAll(NodeType.Restserver)
+          .map(_.sortBy(_.updated).lastOption)
+      )
+      url = serverNode.map(_.url).map(u => "DSC_DOCSPELL_URL" -> u.asString)
+      secret = serverNode.flatMap(_.serverSecret)
+
+      token <- AuthToken.user(
+        accountId,
+        false,
+        secret.getOrElse(ByteVector.empty),
+        tokenValidity.some
+      )
+      session = ("DSC_SESSION" -> token.asString).some
+    } yield List(url, session).flatten.toMap
+}

modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.joex
+
+import docspell.backend.joex.AddonOps.AddonRunConfigRef
+import docspell.logging.Logger
+
+trait LoggerExtension {
+
+  implicit final class LoggerDataOps[F[_]](self: Logger[F]) {
+    def withRunConfig(t: AddonRunConfigRef): Logger[F] =
+      self.capture("addon-task-id", t.id)
+  }
+}

modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import cats.data.NonEmptyList
+
+import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType}
+
+sealed trait AddonRunConfigError {
+  final def cast: AddonRunConfigError = this
+
+  def toLeft[A]: Either[AddonRunConfigError, A] = Left(this)
+
+  def message: String
+}
+
+object AddonRunConfigError {
+
+  case object MissingSchedule extends AddonRunConfigError {
+    val message =
+      "The run config has a trigger 'scheduled' but doesn't provide a schedule!"
+  }
+
+  case object ObsoleteSchedule extends AddonRunConfigError {
+    val message = "The run config has a schedule, but not a trigger 'Scheduled'."
+  }
+
+  case class MismatchingTrigger(unsupported: NonEmptyList[(String, AddonTriggerType)])
+      extends AddonRunConfigError {
+    def message: String = {
+      val list =
+        unsupported.map { case (name, tt) => s"$name: ${tt.name}" }.toList.mkString(", ")
+      s"Some listed addons don't support all defined triggers: $list"
+    }
+  }
+
+  object MismatchingTrigger {
+    def apply(addon: AddonMeta, tt: AddonTriggerType): MismatchingTrigger =
+      MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt))
+
+    def apply(addon: AddonArchive, tt: AddonTriggerType): MismatchingTrigger =
+      MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt))
+  }
+}

modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import cats.data.NonEmptyList
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.backend.ops.AddonRunConfigError._
+import docspell.backend.ops.OAddons.{AddonRunConfigResult, AddonRunInsert}
+import docspell.common.Ident
+import docspell.store.Store
+import docspell.store.records.RAddonArchive
+
+object AddonRunConfigValidate {
+
+  def apply[F[_]: Sync](store: Store[F], cid: Ident)(
+      cfg: AddonRunInsert
+  ): F[AddonRunConfigResult[AddonRunInsert]] = {
+    val init: AddonRunConfigResult[Unit] = ().asRight
+
+    List(
+      checkScheduled(cfg).pure[F],
+      checkTriggers(store, cid)(cfg)
+    )
+      .foldLeftM(init)((res, fr) => fr.map(r => res.flatMap(_ => r)))
+      .map(_.as(cfg))
+  }
+
+  def checkTriggers[F[_]: Sync](store: Store[F], cid: Ident)(
+      cfg: AddonRunInsert
+  ): F[AddonRunConfigResult[Unit]] =
+    for {
+      addons <- store.transact(RAddonArchive.findByIds(cid, cfg.addons.map(_.addonId)))
+      given = cfg.triggered.toList.toSet
+      res = addons
+        .flatMap(r => given.diff(r.triggers).map(tt => r.nameAndVersion -> tt))
+
+      maybeError = NonEmptyList
+        .fromList(res)
+        .map(nel => MismatchingTrigger(nel))
+    } yield maybeError.map(_.toLeft).getOrElse(Right(()))
+
+  def checkScheduled(cfg: AddonRunInsert): AddonRunConfigResult[Unit] =
+    (cfg.isScheduled, cfg.schedule) match {
+      case (true, None)     => MissingSchedule.toLeft[Unit]
+      case (false, Some(_)) => ObsoleteSchedule.toLeft[Unit]
+      case _                => ().asRight
+    }
+}

modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import cats.data.EitherT
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.Path
+
+import docspell.addons.{AddonMeta, RunnerType}
+import docspell.backend.Config
+import docspell.backend.ops.AddonValidationError._
+import docspell.backend.ops.OAddons.AddonValidationResult
+import docspell.common.{Ident, LenientUri, UrlReader}
+import docspell.joexapi.model.AddonSupport
+import docspell.store.Store
+import docspell.store.records.RAddonArchive
+
+final class AddonValidate[F[_]: Async](
+    cfg: Config.Addons,
+    store: Store[F],
+    joexOps: OJoex[F]
+) {
+  private[this] val logger = docspell.logging.getLogger[F]
+
+  def fromUrl(
+      collective: Ident,
+      url: LenientUri,
+      reader: UrlReader[F],
+      localUrl: Option[LenientUri] = None,
+      checkExisting: Boolean = true
+  ): F[AddonValidationResult[AddonMeta]] =
+    if (!cfg.enabled) AddonsDisabled.resultF
+    else if (cfg.isDenied(url)) UrlUntrusted(url).resultF
+    else if (checkExisting)
+      store.transact(RAddonArchive.findByUrl(collective, url)).flatMap {
+        case Some(ar) =>
+          AddonExists("An addon with this url already exists!", ar).resultF
+        case None =>
+          archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting)
+      }
+    else archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting)
+
+  def archive(
+      collective: Ident,
+      addonData: Either[Path, Stream[F, Byte]],
+      checkExisting: Boolean = true
+  ): F[AddonValidationResult[AddonMeta]] =
+    (for {
+      _ <- EitherT.cond[F](cfg.enabled, (), AddonsDisabled.cast)
+
+      meta <-
+        EitherT(
+          addonData
+            .fold(
+              AddonMeta.findInDirectory[F],
+              AddonMeta.findInZip[F]
+            )
+            .attempt
+        )
+          .leftMap(ex => NotAnAddon(ex).cast)
+      _ <- EitherT.cond(
+        meta.triggers.exists(_.nonEmpty),
+        (),
+        InvalidAddon(
+          "The addon doesn't define any triggers. At least one is required!"
+        ).cast
+      )
+      _ <- EitherT.cond(
+        meta.options.exists(_.isUseful),
+        (),
+        InvalidAddon(
+          "Addon defines no output and no networking. It can't do anything useful."
+        ).cast
+      )
+      _ <- EitherT.cond(cfg.allowImpure || meta.isPure, (), ImpureAddonsDisabled.cast)
+
+      _ <-
+        if (checkExisting)
+          EitherT(
+            store
+              .transact(
+                RAddonArchive
+                  .findByNameAndVersion(collective, meta.meta.name, meta.meta.version)
+              )
+              .map {
+                case Some(ar) => AddonExists(ar).result
+                case None     => rightUnit
+              }
+          )
+        else rightUnitT
+
+      joexSupport <- EitherT.liftF(joexOps.getAddonSupport)
+      addonRunners <- EitherT.liftF(meta.enabledTypes(addonData))
+      _ <- EitherT.liftF(
+        logger.info(
+          s"Comparing joex support vs addon runner: $joexSupport vs. $addonRunners"
+        )
+      )
+      _ <- EitherT.fromEither(validateJoexSupport(addonRunners, joexSupport))
+
+    } yield meta).value
+
+  private def validateJoexSupport(
+      addonRunnerTypes: List[RunnerType],
+      joexSupport: List[AddonSupport]
+  ): AddonValidationResult[Unit] = {
+    val addonRunners = addonRunnerTypes.mkString(", ")
+    for {
+      _ <- Either.cond(
+        joexSupport.nonEmpty,
+        (),
+        AddonUnsupported("There are no joex nodes that have addons enabled!", Nil).cast
+      )
+      _ <- Either.cond(
+        addonRunners.nonEmpty,
+        (),
+        InvalidAddon("The addon doesn't enable any runner.")
+      )
+
+      ids = joexSupport
+        .map(n => n.nodeId -> n.runners.intersect(addonRunnerTypes).toSet)
+
+      unsupportedJoex = ids.filter(_._2.isEmpty).map(_._1)
+
+      _ <- Either.cond(
+        ids.forall(_._2.nonEmpty),
+        (),
+        AddonUnsupported(
+          s"A joex node doesn't support this addons runners: $addonRunners. " +
+            s"Check: ${unsupportedJoex.map(_.id).mkString(", ")}.",
+          unsupportedJoex
+        ).cast
+      )
+    } yield ()
+  }
+
+  private def rightUnit: AddonValidationResult[Unit] =
+    ().asRight[AddonValidationError]
+
+  private def rightUnitT: EitherT[F, AddonValidationError, Unit] =
+    EitherT.fromEither(rightUnit)
+
+  implicit final class ErrorOps(self: AddonValidationError) {
+    def result: AddonValidationResult[AddonMeta] =
+      self.toLeft
+
+    def resultF: F[AddonValidationResult[AddonMeta]] =
+      result.pure[F]
+  }
+}

modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import docspell.common.{Ident, LenientUri}
+import docspell.store.records.RAddonArchive
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+sealed trait AddonValidationError {
+  def cast: AddonValidationError = this
+
+  def toLeft[A]: Either[AddonValidationError, A] = Left(this)
+}
+
+object AddonValidationError {
+
+  implicit private val throwableDecoder: Decoder[Throwable] =
+    Decoder.decodeString.map(new Exception(_))
+  implicit private val throwableEncoder: Encoder[Throwable] =
+    Encoder.encodeString.contramap(_.getMessage)
+
+  case object AddonsDisabled extends AddonValidationError {}
+
+  case class UrlUntrusted(url: LenientUri) extends AddonValidationError
+  object UrlUntrusted {
+    implicit val jsonDecoder: Decoder[UrlUntrusted] = deriveDecoder
+    implicit val jsonEncoder: Encoder[UrlUntrusted] = deriveEncoder
+  }
+
+  case class NotAnAddon(error: Throwable) extends AddonValidationError
+  object NotAnAddon {
+    implicit val jsonDecoder: Decoder[NotAnAddon] = deriveDecoder
+    implicit val jsonEncoder: Encoder[NotAnAddon] = deriveEncoder
+  }
+
+  case class AddonUnsupported(message: String, affectedNodes: List[Ident])
+      extends AddonValidationError
+  object AddonUnsupported {
+    implicit val jsonDecoder: Decoder[AddonUnsupported] = deriveDecoder
+    implicit val jsonEncoder: Encoder[AddonUnsupported] = deriveEncoder
+  }
+
+  case class InvalidAddon(message: String) extends AddonValidationError
+  object InvalidAddon {
+    implicit val jsonDecoder: Decoder[InvalidAddon] = deriveDecoder
+    implicit val jsonEncoder: Encoder[InvalidAddon] = deriveEncoder
+  }
+
+  case class AddonExists(message: String, addon: RAddonArchive)
+      extends AddonValidationError
+  object AddonExists {
+    def apply(addon: RAddonArchive): AddonExists =
+      AddonExists(s"An addon '${addon.name}/${addon.version}' already exists!", addon)
+
+    implicit val jsonDecoder: Decoder[AddonExists] = deriveDecoder
+    implicit val jsonEncoder: Encoder[AddonExists] = deriveEncoder
+  }
+
+  case object AddonNotFound extends AddonValidationError
+
+  case class DownloadFailed(error: Throwable) extends AddonValidationError
+  object DownloadFailed {
+    implicit val jsonDecoder: Decoder[DownloadFailed] = deriveDecoder
+    implicit val jsonEncoder: Encoder[DownloadFailed] = deriveEncoder
+  }
+
+  case object ImpureAddonsDisabled extends AddonValidationError
+
+  case object RefreshLocalAddon extends AddonValidationError
+
+  implicit val jsonConfig: Configuration =
+    Configuration.default.withKebabCaseConstructorNames
+      .withDiscriminator("errorType")
+
+  implicit val jsonDecoder: Decoder[AddonValidationError] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AddonValidationError] = deriveConfiguredEncoder
+}

modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala

@@ -0,0 +1,426 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import cats.data.{EitherT, NonEmptyList, OptionT}
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.addons.{AddonMeta, AddonTriggerType}
+import docspell.backend.ops.AddonValidationError._
+import docspell.backend.ops.OAddons._
+import docspell.backend.{Config, JobFactory}
+import docspell.common._
+import docspell.logging.Logger
+import docspell.scheduler.JobStore
+import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore}
+import docspell.store.Store
+import docspell.store.file.FileUrlReader
+import docspell.store.records._
+
+import com.github.eikek.calev.CalEvent
+
+trait OAddons[F[_]] {
+
+  /** Registers a new addon. An error is returned if an addon with this url already
+    * exists.
+    */
+  def registerAddon(
+      collective: Ident,
+      url: LenientUri,
+      logger: Option[Logger[F]]
+  ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]]
+
+  /** Refreshes an existing addon by downloading it again and updating metadata. */
+  def refreshAddon(
+      collective: Ident,
+      addonId: Ident
+  ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]]
+
+  /** Look into the addon at the given url and return its metadata. */
+  def inspectAddon(
+      collective: Ident,
+      url: LenientUri
+  ): F[AddonValidationResult[AddonMeta]]
+
+  /** Deletes the addon if it exists. */
+  def deleteAddon(collective: Ident, addonId: Ident): F[Boolean]
+
+  def getAllAddons(collective: Ident): F[List[RAddonArchive]]
+
+  /** Inserts or updates the addon run configuration. If it already exists (and the given
+    * id is non empty), it will be completely replaced with the given one.
+    */
+  def upsertAddonRunConfig(
+      collective: Ident,
+      runConfig: AddonRunInsert
+  ): F[AddonRunConfigResult[Ident]]
+
+  /** Deletes this task from the database. */
+  def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean]
+
+  def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]]
+
+  def runAddonForItem(
+      account: AccountId,
+      itemIds: NonEmptyList[Ident],
+      addonRunConfigIds: Set[Ident]
+  ): F[Unit]
+}
+
+object OAddons {
+  val scheduledAddonTaskName: Ident =
+    ScheduledAddonTaskArgs.taskName
+
+  case class AddonRunInsert(
+      id: Ident,
+      name: String,
+      enabled: Boolean,
+      userId: Option[Ident],
+      schedule: Option[CalEvent],
+      triggered: NonEmptyList[AddonTriggerType],
+      addons: NonEmptyList[AddonArgs]
+  ) {
+
+    def isScheduled: Boolean =
+      triggered.exists(_ == AddonTriggerType.Scheduled)
+  }
+  case class AddonArgs(addonId: Ident, args: String)
+
+  case class AddonRunInfo(
+      id: Ident,
+      name: String,
+      enabled: Boolean,
+      userId: Option[Ident],
+      schedule: Option[CalEvent],
+      triggered: List[AddonTriggerType],
+      addons: List[(RAddonArchive, RAddonRunConfigAddon)]
+  )
+  object AddonRunInfo {
+    def fromRunConfigData(
+        timer: Option[CalEvent],
+        addons: List[(RAddonArchive, RAddonRunConfigAddon)]
+    )(t: AddonRunConfigData): AddonRunInfo =
+      AddonRunInfo(
+        id = t.runConfig.id,
+        name = t.runConfig.name,
+        enabled = t.runConfig.enabled,
+        userId = t.runConfig.userId,
+        schedule = timer,
+        triggered = t.triggers.map(_.trigger),
+        addons = addons
+      )
+  }
+
+  type AddonRunConfigResult[A] = Either[AddonRunConfigError, A]
+  object AddonRunConfigResult {
+    def success[A](value: A): AddonRunConfigResult[A] = Right(value)
+    def failure[A](error: AddonRunConfigError): AddonRunConfigResult[A] = error.toLeft[A]
+  }
+
+  type AddonValidationResult[A] = Either[AddonValidationError, A]
+  object AddonValidationResult {
+    def success[A](value: A): AddonValidationResult[A] = Right(value)
+    def failure[A](error: AddonValidationError): AddonValidationResult[A] = Left(error)
+  }
+
+  def apply[F[_]: Async](
+      cfg: Config.Addons,
+      store: Store[F],
+      userTasks: UserTaskStore[F],
+      jobStore: JobStore[F],
+      joex: OJoex[F]
+  ): OAddons[F] =
+    new OAddons[F] {
+      private[this] val logger = docspell.logging.getLogger[F]
+      private val urlReader = FileUrlReader(store.fileRepo)
+      private val zip = MimeType.zip.asString
+      private val addonValidate = new AddonValidate[F](cfg, store, joex)
+
+      def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] =
+        for {
+          all <- store.transact(AddonRunConfigData.findAll(collective))
+          runConfigIDs = all.map(_.runConfig.id).toSet
+          archiveIds = all.flatMap(_.addons.map(_.addonId)).distinct
+          archives <- NonEmptyList
+            .fromList(archiveIds)
+            .fold(List.empty[RAddonArchive].pure[F])(ids =>
+              store.transact(RAddonArchive.findByIds(collective, ids))
+            )
+          archivesMap = archives.groupBy(_.id)
+          ptask <- userTasks
+            .getAll(UserTaskScope.collective(collective))
+            .filter(ut => runConfigIDs.contains(ut.id))
+            .map(ut => ut.id -> ut)
+            .compile
+            .toList
+            .map(_.toMap)
+          result = all.map { t =>
+            AddonRunInfo.fromRunConfigData(
+              ptask.get(t.runConfig.id).map(_.timer),
+              t.addons.map(raa => (archivesMap(raa.addonId).head, raa))
+            )(t)
+          }
+        } yield result
+
+      def upsertAddonRunConfig(
+          collective: Ident,
+          runConfig: AddonRunInsert
+      ): F[AddonRunConfigResult[Ident]] = {
+        val insertDataRaw = AddonRunConfigData(
+          RAddonRunConfig(
+            runConfig.id,
+            collective,
+            runConfig.userId,
+            runConfig.name,
+            runConfig.enabled,
+            Timestamp.Epoch
+          ),
+          runConfig.addons.zipWithIndex.map { case (a, index) =>
+            RAddonRunConfigAddon(Ident.unsafe(""), runConfig.id, a.addonId, a.args, index)
+          }.toList,
+          runConfig.triggered
+            .map(t => RAddonRunConfigTrigger(Ident.unsafe(""), runConfig.id, t))
+            .toList
+        )
+
+        val upsert = for {
+          userId <-
+            OptionT
+              .fromOption(runConfig.userId)
+              .flatMapF(uid => store.transact(RUser.getIdByIdOrLogin(uid)))
+              .map(_.uid)
+              .value
+          insertData =
+            insertDataRaw.copy(runConfig =
+              insertDataRaw.runConfig.copy(userId = userId.orElse(runConfig.userId))
+            )
+          id <-
+            OptionT(store.transact(RAddonRunConfig.findById(collective, runConfig.id)))
+              .map(rt =>
+                AddonRunConfigData(
+                  rt.copy(
+                    userId = insertData.runConfig.userId,
+                    name = insertData.runConfig.name,
+                    enabled = insertData.runConfig.enabled
+                  ),
+                  insertData.addons,
+                  insertData.triggers
+                )
+              )
+              .semiflatMap(rt =>
+                store.transact(AddonRunConfigData.update(rt).as(rt.runConfig.id))
+              )
+              .getOrElseF(store.transact(AddonRunConfigData.insert(insertData)))
+        } yield id
+
+        EitherT(AddonRunConfigValidate(store, collective)(runConfig))
+          .semiflatMap(_ =>
+            upsert.flatTap { runConfigId =>
+              runConfig.schedule match {
+                case Some(timer) =>
+                  userTasks.updateTask(
+                    UserTaskScope.collective(collective),
+                    s"Addon task ${runConfig.name}".some,
+                    UserTask(
+                      runConfigId,
+                      scheduledAddonTaskName,
+                      true,
+                      timer,
+                      s"Running scheduled addon task ${runConfig.name}".some,
+                      ScheduledAddonTaskArgs(collective, runConfigId)
+                    )
+                  )
+                case None =>
+                  userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId)
+              }
+            }
+          )
+          .value
+      }
+
+      def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] = {
+        val deleteRunConfig =
+          (for {
+            e <- OptionT(RAddonRunConfig.findById(collective, runConfigId))
+            _ <- OptionT.liftF(RAddonRunConfigAddon.deleteAllForConfig(e.id))
+            _ <- OptionT.liftF(RAddonRunConfigTrigger.deleteAllForConfig(e.id))
+            _ <- OptionT.liftF(RAddonRunConfig.deleteById(collective, e.id))
+          } yield true).getOrElse(false)
+
+        for {
+          deleted <- store.transact(deleteRunConfig)
+          _ <-
+            if (deleted)
+              userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId)
+            else 0.pure[F]
+        } yield deleted
+      }
+
+      def getAllAddons(collective: Ident): F[List[RAddonArchive]] =
+        store.transact(RAddonArchive.listAll(collective))
+
+      def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] =
+        store.transact(RAddonArchive.deleteById(collective, addonId)).map(_ > 0)
+
+      def inspectAddon(
+          collective: Ident,
+          url: LenientUri
+      ): F[AddonValidationResult[AddonMeta]] =
+        addonValidate.fromUrl(collective, url, urlReader, checkExisting = false)
+
+      def registerAddon(
+          collective: Ident,
+          url: LenientUri,
+          logger: Option[Logger[F]]
+      ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = {
+        val log = logger.getOrElse(this.logger)
+        def validateAndInsert(file: FileKey, localUrl: LenientUri) =
+          addonValidate.fromUrl(collective, url, urlReader, localUrl.some).flatMap {
+            case Right(meta) =>
+              insertAddon(collective, url, meta, file)
+                .map(ar => AddonValidationResult.success(ar -> meta))
+
+            case Left(error) =>
+              store.fileRepo
+                .delete(file)
+                .as(AddonValidationResult.failure[(RAddonArchive, AddonMeta)](error))
+          }
+
+        log.info(s"Store addon file from '${url.asString} for ${collective.id}") *>
+          storeAddonFromUrl(collective, url).flatMapF { file =>
+            val localUrl = FileUrlReader.url(file)
+            for {
+              _ <- log.info(s"Validating addon…")
+              res <- validateAndInsert(file, localUrl)
+              _ <- log.info(s"Validation result: $res")
+            } yield res
+          }.value
+      }
+
+      def refreshAddon(
+          collective: Ident,
+          addonId: Ident
+      ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = {
+        val findAddon = store
+          .transact(RAddonArchive.findById(collective, addonId))
+          .map(_.toRight(AddonNotFound))
+        def validateAddon(aa: RAddonArchive): F[AddonValidationResult[AddonMeta]] =
+          aa.originalUrl.fold(
+            AddonValidationResult.failure[AddonMeta](RefreshLocalAddon).pure[F]
+          )(url =>
+            addonValidate.fromUrl(collective, url, urlReader, checkExisting = false)
+          )
+
+        EitherT(findAddon).flatMap { aa =>
+          EitherT(validateAddon(aa))
+            .flatMap(meta => refreshAddon(aa, meta).map(na => na -> meta))
+        }.value
+      }
+
+      private def refreshAddon(
+          r: RAddonArchive,
+          meta: AddonMeta
+      ): EitherT[F, AddonValidationError, RAddonArchive] =
+        if (r.isUnchanged(meta)) EitherT.pure(r)
+        else
+          r.originalUrl match {
+            case Some(url) =>
+              EitherT(
+                store
+                  .transact(
+                    RAddonArchive
+                      .findByNameAndVersion(r.cid, meta.meta.name, meta.meta.version)
+                  )
+                  .map(
+                    _.fold(().asRight[AddonValidationError])(rx => AddonExists(rx).toLeft)
+                  )
+              ).flatMap(_ =>
+                storeAddonFromUrl(r.cid, url).flatMap { file =>
+                  val nr = r.update(file, meta)
+                  for {
+                    _ <- EitherT(
+                      store
+                        .transact(RAddonArchive.update(nr))
+                        .map(_.asRight[AddonValidationError])
+                        .recoverWith { case ex =>
+                          logger.warn(ex)(s"Storing addon metadata failed.") *>
+                            store.fileRepo
+                              .delete(file)
+                              .as(
+                                AddonExists(
+                                  s"The addon '${nr.name}/${nr.version}' could not be stored",
+                                  nr
+                                ).toLeft
+                              )
+                        }
+                    )
+                    _ <- EitherT.liftF(store.fileRepo.delete(r.fileId))
+                  } yield nr
+                }
+              )
+            case None =>
+              EitherT.leftT(RefreshLocalAddon.cast)
+          }
+
+      private def insertAddon(
+          collective: Ident,
+          url: LenientUri,
+          meta: AddonMeta,
+          file: FileKey
+      ): F[RAddonArchive] =
+        for {
+          now <- Timestamp.current[F]
+          aId <- Ident.randomId[F]
+          record = RAddonArchive(
+            aId,
+            collective,
+            file,
+            url.some,
+            meta,
+            now
+          )
+          _ <- store
+            .transact(RAddonArchive.insert(record, silent = false))
+            .onError(_ => store.fileRepo.delete(file))
+        } yield record
+
+      private def storeAddonFromUrl(collective: Ident, url: LenientUri) =
+        for {
+          urlFile <- EitherT.pure(url.path.segments.lastOption)
+          file <- EitherT(
+            urlReader(url)
+              .through(
+                store.fileRepo.save(
+                  collective,
+                  FileCategory.Addon,
+                  MimeTypeHint(urlFile, zip.some)
+                )
+              )
+              .compile
+              .lastOrError
+              .attempt
+              .map(_.leftMap(DownloadFailed(_).cast))
+          )
+        } yield file
+
+      def runAddonForItem(
+          account: AccountId,
+          itemIds: NonEmptyList[Ident],
+          addonRunConfigIds: Set[Ident]
+      ): F[Unit] =
+        for {
+          jobs <- itemIds.traverse(id =>
+            JobFactory.existingItemAddon(
+              ItemAddonTaskArgs(account.collective, id, addonRunConfigIds),
+              account
+            )
+          )
+          _ <- jobStore.insertAllIfNew(jobs.map(_.encode).toList)
+        } yield ()
+    }
+}

modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala

@@ -0,0 +1,223 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.backend.ops
+
+import cats.data.{NonEmptyList => Nel, OptionT}
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+
+import docspell.backend.JobFactory
+import docspell.common.MakePreviewArgs.StoreMode
+import docspell.common._
+import docspell.files.TikaMimetype
+import docspell.ftsclient.{FtsClient, TextData}
+import docspell.scheduler.JobStore
+import docspell.store.Store
+import docspell.store.queries.QAttachment
+import docspell.store.records._
+
+trait OAttachment[F[_]] {
+
+  def setExtractedText(
+      collective: Ident,
+      itemId: Ident,
+      attachId: Ident,
+      newText: F[String]
+  ): F[Unit]
+
+  def addOrReplacePdf(
+      collective: Ident,
+      attachId: Ident,
+      pdfData: Stream[F, Byte],
+      regeneratePreview: Boolean
+  ): F[Unit]
+
+  def addOrReplacePreview(
+      collective: Ident,
+      attachId: Ident,
+      imageData: Stream[F, Byte]
+  ): F[Unit]
+}
+
+object OAttachment {
+
+  def apply[F[_]: Sync](
+      store: Store[F],
+      fts: FtsClient[F],
+      jobStore: JobStore[F]
+  ): OAttachment[F] =
+    new OAttachment[F] {
+      private[this] val logger = docspell.logging.getLogger[F]
+
+      def setExtractedText(
+          collective: Ident,
+          itemId: Ident,
+          attachId: Ident,
+          newText: F[String]
+      ): F[Unit] =
+        for {
+          _ <- logger.info(s"Find attachment ${attachId.id} to update extracted text.")
+          cca <- store
+            .transact(
+              QAttachment
+                .allAttachmentMetaAndName(
+                  collective.some,
+                  Nel.of(itemId).some,
+                  ItemState.validStates.append(ItemState.Processing),
+                  100
+                )
+            )
+            .filter(_.id == attachId)
+            .compile
+            .last
+          content = cca.find(_.id == attachId)
+          _ <- logger.debug(s"Found existing metadata: ${content.isDefined}")
+          _ <- OptionT
+            .fromOption(content)
+            .semiflatMap { cnt =>
+              for {
+                _ <- logger.debug(s"Setting new extracted text on ${cnt.id.id}")
+                text <- newText
+                td = TextData.attachment(
+                  cnt.item,
+                  cnt.id,
+                  cnt.collective,
+                  cnt.folder,
+                  cnt.lang,
+                  cnt.name,
+                  text.some
+                )
+                _ <- store.transact(RAttachmentMeta.updateContent(attachId, text))
+                _ <- fts.updateIndex(logger, td)
+              } yield ()
+            }
+            .getOrElseF(
+              logger.warn(
+                s"Item or attachment meta not found to update text: ${itemId.id}"
+              )
+            )
+        } yield ()
+
+      def addOrReplacePdf(
+          collective: Ident,
+          attachId: Ident,
+          pdfData: Stream[F, Byte],
+          regeneratePreview: Boolean
+      ): F[Unit] = {
+        def generatePreview(ra: RAttachment): F[Unit] =
+          JobFactory
+            .makePreview(MakePreviewArgs(ra.id, StoreMode.Replace), None)
+            .map(_.encode)
+            .flatMap(jobStore.insert) *>
+            logger.info(s"Job submitted to re-generate preview from new pdf")
+
+        def generatePageCount(ra: RAttachment): F[Unit] =
+          JobFactory
+            .makePageCount(
+              MakePageCountArgs(ra.id),
+              AccountId(collective, DocspellSystem.user).some
+            )
+            .map(_.encode)
+            .flatMap(jobStore.insert) *>
+            logger.info(s"Job submitted to find page count from new pdf")
+
+        def setFile(ra: RAttachment, rs: RAttachmentSource) =
+          for {
+            _ <- requireMimeType(pdfData, MimeType.pdf)
+
+            newFile <- pdfData
+              .through(
+                store.fileRepo.save(
+                  collective,
+                  FileCategory.AttachmentConvert,
+                  MimeTypeHint.advertised(MimeType.pdf)
+                )
+              )
+              .compile
+              .lastOrError
+
+            _ <- store.transact(RAttachment.updateFileId(attachId, newFile))
+            _ <- logger.info(s"Deleting old file for attachment")
+            _ <-
+              if (rs.fileId == ra.fileId) ().pure[F]
+              else store.fileRepo.delete(ra.fileId)
+            _ <-
+              if (regeneratePreview) generatePreview(ra)
+              else ().pure[F]
+            _ <- generatePageCount(ra)
+          } yield ()
+
+        (for {
+          ra <- OptionT(
+            store.transact(RAttachment.findByIdAndCollective(attachId, collective))
+          )
+          rs <- OptionT(
+            store.transact(RAttachmentSource.findByIdAndCollective(attachId, collective))
+          )
+          _ <- OptionT.liftF(setFile(ra, rs))
+        } yield ()).getOrElseF(
+          logger.warn(
+            s"Cannot replace pdf file. Attachment not found for id: ${attachId.id}"
+          )
+        )
+      }
+
+      def addOrReplacePreview(
+          collective: Ident,
+          attachId: Ident,
+          imageData: Stream[F, Byte]
+      ): F[Unit] = {
+        def setFile(ra: RAttachment): F[Unit] =
+          for {
+            _ <- requireMimeType(imageData, MimeType.image("*"))
+            newFile <- imageData
+              .through(
+                store.fileRepo
+                  .save(collective, FileCategory.PreviewImage, MimeTypeHint.none)
+              )
+              .compile
+              .lastOrError
+
+            now <- Timestamp.current[F]
+            record = RAttachmentPreview(ra.id, newFile, None, now)
+            oldFile <- store.transact(RAttachmentPreview.upsert(record))
+            _ <- OptionT
+              .fromOption(oldFile)
+              .semiflatMap(store.fileRepo.delete)
+              .getOrElse(())
+          } yield ()
+
+        (for {
+          ra <- OptionT(
+            store.transact(RAttachment.findByIdAndCollective(attachId, collective))
+          )
+          _ <- OptionT.liftF(setFile(ra))
+        } yield ()).getOrElseF(
+          logger.warn(
+            s"Cannot add/replace preview file. Attachment not found for id: ${attachId.id}"
+          )
+        )
+      }
+    }
+
+  private def requireMimeType[F[_]: Sync](
+      data: Stream[F, Byte],
+      expectedMime: MimeType
+  ): F[Unit] =
+    TikaMimetype
+      .detect(data, MimeTypeHint.advertised(expectedMime))
+      .flatMap { mime =>
+        if (expectedMime.matches(mime)) ().pure[F]
+        else
+          Sync[F].raiseError(
+            new IllegalArgumentException(
+              s"Expected pdf file, but got: ${mime.asString}"
+            )
+          )
+      }
+}

modules/backend/src/main/scala/docspell/backend/ops/OItem.scala

@@ -61,6 +61,12 @@ trait OItem[F[_]] {
       collective: Ident
   ): F[AttachedEvent[UpdateResult]]
 
+  def removeTagsOfCategories(
+      item: Ident,
+      collective: Ident,
+      categories: Set[String]
+  ): F[AttachedEvent[UpdateResult]]
+
   def removeTagsMultipleItems(
       items: Nel[Ident],
       tags: List[String],
@@ -80,11 +86,13 @@ trait OItem[F[_]] {
       collective: Ident
   ): F[UpdateResult]
 
-  def setFolder(item: Ident, folder: Option[Ident], collective: Ident): F[UpdateResult]
+  /** Set or remove the folder on an item. Folder can be the id or name. */
+  def setFolder(item: Ident, folder: Option[String], collective: Ident): F[UpdateResult]
 
+  /** Set or remove the folder on multiple items. Folder can be the id or name. */
   def setFolderMultiple(
       items: Nel[Ident],
-      folder: Option[Ident],
+      folder: Option[String],
       collective: Ident
   ): F[UpdateResult]
 
@@ -122,6 +130,13 @@ trait OItem[F[_]] {
 
   def setNotes(item: Ident, notes: Option[String], collective: Ident): F[UpdateResult]
 
+  def addNotes(
+      item: Ident,
+      notes: String,
+      separator: Option[String],
+      collective: Ident
+  ): F[UpdateResult]
+
   def setName(item: Ident, name: String, collective: Ident): F[UpdateResult]
 
   def setNameMultiple(
@@ -288,6 +303,28 @@ object OItem {
                 }
           }
 
+        def removeTagsOfCategories(
+            item: Ident,
+            collective: Ident,
+            categories: Set[String]
+        ): F[AttachedEvent[UpdateResult]] =
+          if (categories.isEmpty) {
+            AttachedEvent.only(UpdateResult.success).pure[F]
+          } else {
+            val dbtask =
+              for {
+                tags <- RTag.findByItem(item)
+                removeTags = tags.filter(_.category.exists(categories.contains))
+                _ <- RTagItem.removeAllTags(item, removeTags.map(_.tagId))
+                mkEvent = Event.TagsChanged
+                  .partial(Nel.of(item), Nil, removeTags.map(_.tagId.id).toList)
+              } yield AttachedEvent(UpdateResult.success)(mkEvent)
+
+            OptionT(store.transact(RItem.checkByIdAndCollective(item, collective)))
+              .semiflatMap(_ => store.transact(dbtask))
+              .getOrElse(AttachedEvent.only(UpdateResult.notFound))
+          }
+
         def removeTagsMultipleItems(
             items: Nel[Ident],
             tags: List[String],
@@ -420,21 +457,27 @@ object OItem {
 
         def setFolder(
             item: Ident,
-            folder: Option[Ident],
+            folder: Option[String],
             collective: Ident
         ): F[UpdateResult] =
-          UpdateResult
-            .fromUpdate(
-              store
-                .transact(RItem.updateFolder(item, collective, folder))
+          for {
+            result <- store.transact(RItem.updateFolder(item, collective, folder)).attempt
+            ures = result.fold(
+              UpdateResult.failure,
+              t => UpdateResult.fromUpdateRows(t._1)
             )
-            .flatTap(
-              onSuccessIgnoreError(fts.updateFolder(logger, item, collective, folder))
+            _ <- result.fold(
+              _ => ().pure[F],
+              t =>
+                onSuccessIgnoreError(fts.updateFolder(logger, item, collective, t._2))(
+                  ures
+                )
             )
+          } yield ures
 
         def setFolderMultiple(
             items: Nel[Ident],
-            folder: Option[Ident],
+            folder: Option[String],
             collective: Ident
         ): F[UpdateResult] =
           for {
@@ -615,6 +658,33 @@ object OItem {
               }
             )
 
+        def addNotes(
+            item: Ident,
+            notes: String,
+            separator: Option[String],
+            collective: Ident
+        ): F[UpdateResult] =
+          store
+            .transact(RItem.appendNotes(item, collective, notes, separator))
+            .flatMap {
+              case Some(newNotes) =>
+                store
+                  .transact(RCollective.findLanguage(collective))
+                  .map(_.getOrElse(Language.English))
+                  .flatMap(lang =>
+                    fts.updateItemNotes(logger, item, collective, lang, newNotes.some)
+                  )
+                  .attempt
+                  .flatMap {
+                    case Right(()) => ().pure[F]
+                    case Left(ex) =>
+                      logger.warn(s"Error updating full-text index: ${ex.getMessage}")
+                  }
+                  .as(UpdateResult.success)
+              case None =>
+                UpdateResult.notFound.pure[F]
+            }
+
         def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] =
           UpdateResult
             .fromUpdate(

modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala

@@ -6,11 +6,13 @@
 
 package docspell.backend.ops
 
-import cats.Applicative
 import cats.effect._
-import cats.implicits._
+import cats.syntax.all._
+import fs2.Stream
 
-import docspell.common.Ident
+import docspell.common.{Ident, NodeType}
+import docspell.joexapi.client.JoexClient
+import docspell.joexapi.model.AddonSupport
 import docspell.pubsub.api.PubSubT
 import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify}
 
@@ -21,10 +23,16 @@ trait OJoex[F[_]] {
   def notifyPeriodicTasks: F[Unit]
 
   def cancelJob(job: Ident, worker: Ident): F[Unit]
+
+  def getAddonSupport: F[List[AddonSupport]]
 }
 
 object OJoex {
-  def apply[F[_]: Applicative](pubSub: PubSubT[F]): Resource[F, OJoex[F]] =
+  def apply[F[_]: Async](
+      pubSub: PubSubT[F],
+      nodes: ONode[F],
+      joexClient: JoexClient[F]
+  ): Resource[F, OJoex[F]] =
     Resource.pure[F, OJoex[F]](new OJoex[F] {
 
       def notifyAllNodes: F[Unit] =
@@ -35,5 +43,17 @@ object OJoex {
 
       def cancelJob(job: Ident, worker: Ident): F[Unit] =
         pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(())
+
+      def getAddonSupport: F[List[AddonSupport]] =
+        for {
+          joex <- nodes.getNodes(NodeType.Joex)
+          conc = math.max(2, Runtime.getRuntime.availableProcessors() - 1)
+          supp <- Stream
+            .emits(joex)
+            .covary[F]
+            .parEvalMap(conc)(n => joexClient.getAddonSupport(n.url))
+            .compile
+            .toList
+        } yield supp
     })
 }

modules/backend/src/main/scala/docspell/backend/ops/ONode.scala

@@ -13,11 +13,27 @@ import docspell.common.{Ident, LenientUri, NodeType}
 import docspell.store.Store
 import docspell.store.records.RNode
 
+import scodec.bits.ByteVector
+
 trait ONode[F[_]] {
 
-  def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit]
+  def register(
+      appId: Ident,
+      nodeType: NodeType,
+      uri: LenientUri,
+      serverSecret: Option[ByteVector]
+  ): F[Unit]
 
   def unregister(appId: Ident): F[Unit]
+
+  def withRegistered(
+      appId: Ident,
+      nodeType: NodeType,
+      uri: LenientUri,
+      serverSecret: Option[ByteVector]
+  ): Resource[F, Unit]
+
+  def getNodes(nodeType: NodeType): F[Vector[RNode]]
 }
 
 object ONode {
@@ -25,9 +41,14 @@ object ONode {
   def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] =
     Resource.pure[F, ONode[F]](new ONode[F] {
       val logger = docspell.logging.getLogger[F]
-      def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =
+      def register(
+          appId: Ident,
+          nodeType: NodeType,
+          uri: LenientUri,
+          serverSecret: Option[ByteVector]
+      ): F[Unit] =
         for {
-          node <- RNode(appId, nodeType, uri)
+          node <- RNode(appId, nodeType, uri, serverSecret)
           _ <- logger.info(s"Registering node ${node.id.id}")
           _ <- store.transact(RNode.set(node))
         } yield ()
@@ -35,6 +56,19 @@ object ONode {
       def unregister(appId: Ident): F[Unit] =
         logger.info(s"Unregister app ${appId.id}") *>
           store.transact(RNode.delete(appId)).map(_ => ())
+
+      def withRegistered(
+          appId: Ident,
+          nodeType: NodeType,
+          uri: LenientUri,
+          serverSecret: Option[ByteVector]
+      ): Resource[F, Unit] =
+        Resource.make(register(appId, nodeType, uri, serverSecret))(_ =>
+          unregister(appId)
+        )
+
+      def getNodes(nodeType: NodeType): F[Vector[RNode]] =
+        store.transact(RNode.findAll(nodeType))
     })
 
 }

modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala

@@ -9,8 +9,9 @@ package docspell.common
 import java.time.Instant
 
 import io.circe._
+import scodec.bits.ByteVector
 
-object BaseJsonCodecs {
+trait BaseJsonCodecs {
 
   implicit val encodeInstantEpoch: Encoder[Instant] =
     Encoder.encodeJavaLong.contramap(_.toEpochMilli)
@@ -18,4 +19,11 @@ object BaseJsonCodecs {
   implicit val decodeInstantEpoch: Decoder[Instant] =
     Decoder.decodeLong.map(Instant.ofEpochMilli)
 
+  implicit val byteVectorEncoder: Encoder[ByteVector] =
+    Encoder.encodeString.contramap(_.toBase64)
+
+  implicit val byteVectorDecoder: Decoder[ByteVector] =
+    Decoder.decodeString.emap(ByteVector.fromBase64Descriptive(_))
 }
+
+object BaseJsonCodecs extends BaseJsonCodecs

modules/common/src/main/scala/docspell/common/Binary.scala

@@ -18,6 +18,18 @@ final case class Binary[F[_]](name: String, mime: MimeType, data: Stream[F, Byte
 
   def withMime(mime: MimeType): Binary[F] =
     copy(mime = mime)
+
+  /** Return the extension of `name` if available (without the dot) */
+  def extension: Option[String] =
+    name.lastIndexOf('.') match {
+      case n if n > 0 =>
+        Some(name.substring(n + 1))
+      case _ =>
+        None
+    }
+
+  def extensionIn(extensions: Set[String]): Boolean =
+    extension.exists(extensions.contains)
 }
 
 object Binary {

modules/common/src/main/scala/docspell/common/FileCategory.scala

@@ -32,6 +32,7 @@ object FileCategory {
   case object PreviewImage extends FileCategory
   case object Classifier extends FileCategory
   case object DownloadAll extends FileCategory
+  case object Addon extends FileCategory
 
   val all: NonEmptyList[FileCategory] =
     NonEmptyList.of(
@@ -39,7 +40,8 @@ object FileCategory {
       AttachmentConvert,
       PreviewImage,
       Classifier,
-      DownloadAll
+      DownloadAll,
+      Addon
     )
 
   def fromString(str: String): Either[String, FileCategory] =

modules/common/src/main/scala/docspell/common/Glob.scala

@@ -32,7 +32,8 @@ object Glob {
     def single(str: String) =
       PatternGlob(Pattern(split(str, separator).map(makeSegment)))
 
-    if (in == "*") all
+    if (in == all.asString) all
+    else if (in == none.asString) none
     else
       split(in, anyChar) match {
         case NonEmptyList(_, Nil) =>
@@ -51,15 +52,25 @@ object Glob {
     val asString = "*"
   }
 
+  val none = new Glob {
+    def matches(caseSensitive: Boolean)(in: String) = false
+    def matchFilenameOrPath(in: String) = false
+    def asString = "!*"
+  }
+
   def pattern(pattern: Pattern): Glob =
     PatternGlob(pattern)
 
   /** A simple glob supporting `*` and `?`. */
   final private case class PatternGlob(pattern: Pattern) extends Glob {
-    def matches(caseSensitive: Boolean)(in: String): Boolean =
+    def matches(caseSensitive: Boolean)(in: String): Boolean = {
+      val input = Glob.split(in, Glob.separator)
+
+      pattern.parts.size == input.size &&
       pattern.parts
-        .zipWith(Glob.split(in, Glob.separator))(_.matches(caseSensitive)(_))
+        .zipWith(input)(_.matches(caseSensitive)(_))
         .forall(identity)
+    }
 
     def matchFilenameOrPath(in: String): Boolean =
       if (pattern.parts.tail.isEmpty) matches(true)(split(in, separator).last)
@@ -67,6 +78,8 @@ object Glob {
 
     def asString: String =
       pattern.asString
+
+    override def toString = s"PatternGlob($asString)"
   }
 
   final private case class AnyGlob(globs: NonEmptyList[Glob]) extends Glob {
@@ -76,6 +89,8 @@ object Glob {
       globs.exists(_.matchFilenameOrPath(in))
     def asString =
       globs.toList.map(_.asString).mkString(anyChar.toString)
+
+    override def toString = s"AnyGlob($globs)"
   }
 
   case class Pattern(parts: NonEmptyList[Segment]) {

modules/common/src/main/scala/docspell/common/Ident.scala

@@ -26,6 +26,9 @@ case class Ident(id: String) {
 
   def /(next: Ident): Ident =
     new Ident(id + Ident.concatChar + next.id)
+
+  def take(n: Int): Ident =
+    new Ident(id.take(n))
 }
 
 object Ident {

modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+/** Arguments to submit a task that runs addons configured for some existing item.
+  *
+  * If `addonTaskIds` is non empty, only these addon tasks are run. Otherwise all addon
+  * tasks that are configured for 'existing-item' are run.
+  */
+final case class ItemAddonTaskArgs(
+    collective: Ident,
+    itemId: Ident,
+    addonRunConfigs: Set[Ident]
+)
+
+object ItemAddonTaskArgs {
+  val taskName: Ident = Ident.unsafe("addon-existing-item")
+
+  implicit val jsonDecoder: Decoder[ItemAddonTaskArgs] = deriveDecoder
+  implicit val jsonEncoder: Encoder[ItemAddonTaskArgs] = deriveEncoder
+}

modules/common/src/main/scala/docspell/common/MimeTypeHint.scala

@@ -6,6 +6,8 @@
 
 package docspell.common
 
+import fs2.io.file.Path
+
 case class MimeTypeHint(filename: Option[String], advertised: Option[String]) {
 
   def withName(name: String): MimeTypeHint =
@@ -21,6 +23,9 @@ object MimeTypeHint {
   def filename(name: String): MimeTypeHint =
     MimeTypeHint(Some(name), None)
 
+  def filename(file: Path): MimeTypeHint =
+    filename(file.fileName.toString)
+
   def advertised(mimeType: MimeType): MimeTypeHint =
     advertised(mimeType.asString)
 

modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala

@@ -17,7 +17,7 @@ import io.circe.generic.semiauto._
   * This task is run for each new file to create a new item from it or to add this file as
   * an attachment to an existing item.
   *
-  * If the `itemId' is set to some value, the item is tried to load to ammend with the
+  * If the `itemId' is set to some value, the item is tried to load to amend with the
   * given files. Otherwise a new item is created.
   *
   * It is also re-used by the 'ReProcessItem' task.

modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+final case class ScheduledAddonTaskArgs(collective: Ident, addonTaskId: Ident)
+
+object ScheduledAddonTaskArgs {
+  val taskName: Ident = Ident.unsafe("addon-scheduled-task")
+
+  implicit val jsonDecoder: Decoder[ScheduledAddonTaskArgs] = deriveDecoder
+  implicit val jsonEncoder: Encoder[ScheduledAddonTaskArgs] = deriveEncoder
+}

modules/common/src/main/scala/docspell/common/SystemCommand.scala

@@ -17,11 +17,23 @@ import cats.implicits._
 import fs2.io.file.Path
 import fs2.{Stream, io, text}
 
+import docspell.common.{exec => newExec}
 import docspell.logging.Logger
 
+// better use `SysCmd` and `SysExec`
 object SystemCommand {
 
-  final case class Config(program: String, args: Seq[String], timeout: Duration) {
+  final case class Config(
+      program: String,
+      args: Seq[String],
+      timeout: Duration,
+      env: Map[String, String] = Map.empty
+  ) {
+
+    def toSysCmd = newExec
+      .SysCmd(program, newExec.Args(args))
+      .withTimeout(timeout)
+      .addEnv(newExec.Env(env))
 
     def mapArgs(f: String => String): Config =
       Config(program, args.map(f), timeout)
@@ -33,6 +45,18 @@ object SystemCommand {
         }
       )
 
+    def withEnv(key: String, value: String): Config =
+      copy(env = env.updated(key, value))
+
+    def addEnv(moreEnv: Map[String, String]): Config =
+      copy(env = env ++ moreEnv)
+
+    def appendArgs(extraArgs: Args): Config =
+      copy(args = args ++ extraArgs.args)
+
+    def appendArgs(extraArgs: Seq[String]): Config =
+      copy(args = args ++ extraArgs)
+
     def toCmd: List[String] =
       program :: args.toList
 
@@ -40,6 +64,45 @@ object SystemCommand {
       toCmd.mkString(" ")
   }
 
+  final case class Args(args: Vector[String]) extends Iterable[String] {
+    override def iterator = args.iterator
+
+    def prepend(a: String): Args = Args(a +: args)
+
+    def prependWhen(flag: Boolean)(a: String): Args =
+      prependOption(Option.when(flag)(a))
+
+    def prependOption(value: Option[String]): Args =
+      value.map(prepend).getOrElse(this)
+
+    def append(a: String, as: String*): Args =
+      Args(args ++ (a +: as.toVector))
+
+    def appendOption(value: Option[String]): Args =
+      value.map(append(_)).getOrElse(this)
+
+    def appendOptionVal(first: String, second: Option[String]): Args =
+      second.map(b => append(first, b)).getOrElse(this)
+
+    def appendWhen(flag: Boolean)(a: String, as: String*): Args =
+      if (flag) append(a, as: _*) else this
+
+    def appendWhenNot(flag: Boolean)(a: String, as: String*): Args =
+      if (!flag) append(a, as: _*) else this
+
+    def append(p: Path): Args =
+      append(p.toString)
+
+    def append(as: Iterable[String]): Args =
+      Args(args ++ as.toVector)
+  }
+  object Args {
+    val empty: Args = Args()
+
+    def apply(as: String*): Args =
+      Args(as.toVector)
+  }
+
   final case class Result(rc: Int, stdout: String, stderr: String)
 
   def exec[F[_]: Sync](
@@ -104,6 +167,10 @@ object SystemCommand {
           .redirectError(Redirect.PIPE)
           .redirectOutput(Redirect.PIPE)
 
+        val pbEnv = pb.environment()
+        cmd.env.foreach { case (key, value) =>
+          pbEnv.put(key, value)
+        }
         wd.map(_.toNioPath.toFile).foreach(pb.directory)
         pb.start()
       }

modules/common/src/main/scala/docspell/common/UrlMatcher.scala

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import cats.data.NonEmptyList
+import cats.kernel.Monoid
+import cats.syntax.all._
+
+trait UrlMatcher {
+  def matches(url: LenientUri): Boolean
+}
+
+object UrlMatcher {
+  val True = instance(_ => true)
+  val False = instance(_ => false)
+
+  def instance(f: LenientUri => Boolean): UrlMatcher =
+    (url: LenientUri) => f(url)
+
+  def fromString(str: String): Either[String, UrlMatcher] =
+    if (str == "") False.asRight
+    else if (str == "*") True.asRight
+    else LenientUri.parse(str).map(fromUrl)
+
+  def unsafeFromString(str: String): UrlMatcher =
+    fromString(str).fold(sys.error, identity)
+
+  def fromStringList(str: List[String]): Either[String, UrlMatcher] =
+    str match {
+      case Nil => False.asRight
+      case _   => str.map(_.trim).traverse(fromString).map(_.combineAll)
+    }
+
+  def fromUrl(url: LenientUri): UrlMatcher = {
+    val schemeGlob = Glob(url.scheme.head)
+    val hostGlob = HostGlob(url.host)
+    val pathGlob = Glob(url.path.asString)
+    new Impl(schemeGlob, hostGlob, pathGlob, url.path.segments.size)
+  }
+
+  def any(ulrm: IterableOnce[UrlMatcher]): UrlMatcher =
+    anyMonoid.combineAll(ulrm)
+
+  def all(urlm: IterableOnce[UrlMatcher]): UrlMatcher =
+    allMonoid.combineAll(urlm)
+
+  val anyMonoid: Monoid[UrlMatcher] =
+    Monoid.instance(False, (a, b) => instance(url => a.matches(url) || b.matches(url)))
+
+  val allMonoid: Monoid[UrlMatcher] =
+    Monoid.instance(True, (a, b) => instance(url => a.matches(url) && b.matches(url)))
+
+  implicit val defaultMonoid: Monoid[UrlMatcher] = anyMonoid
+
+  private class Impl(scheme: Glob, host: HostGlob, path: Glob, pathSegmentCount: Int)
+      extends UrlMatcher {
+    def matches(url: LenientUri) = {
+      // strip path to only match prefixes
+      val mPath: LenientUri.Path =
+        NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match {
+          case Some(nel) => LenientUri.NonEmptyPath(nel)
+          case None      => LenientUri.RootPath
+        }
+
+      url.scheme.forall(scheme.matches(false)) &&
+      host.matches(url.host) &&
+      path.matchFilenameOrPath(mPath.asString)
+    }
+  }
+
+  private class HostGlob(glob: Option[Glob]) {
+    def matches(host: Option[String]): Boolean =
+      (glob, host) match {
+        case (Some(pattern), Some(word)) =>
+          pattern.matches(false)(HostGlob.prepareHost(word))
+        case (None, None) => true
+        case _            => false
+      }
+
+    override def toString = s"HostGlob(${glob.map(_.asString)})"
+  }
+
+  private object HostGlob {
+    def apply(hostPattern: Option[String]): HostGlob =
+      new HostGlob(hostPattern.map(p => Glob(prepareHost(p))))
+
+    private def prepareHost(host: String): String =
+      host.replace('.', '/')
+  }
+}

modules/common/src/main/scala/docspell/common/UrlReader.scala

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import cats.ApplicativeError
+import cats.effect._
+import fs2.Stream
+
+trait UrlReader[F[_]] {
+  def apply(url: LenientUri): Stream[F, Byte]
+}
+
+object UrlReader {
+
+  def instance[F[_]](f: LenientUri => Stream[F, Byte]): UrlReader[F] =
+    (url: LenientUri) => f(url)
+
+  def failWith[F[_]](
+      message: String
+  )(implicit F: ApplicativeError[F, Throwable]): UrlReader[F] =
+    instance(url =>
+      Stream.raiseError(
+        new IllegalStateException(s"Unable to read '${url.asString}': $message")
+      )
+    )
+
+  def apply[F[_]](implicit r: UrlReader[F]): UrlReader[F] = r
+
+  implicit def defaultReader[F[_]: Sync]: UrlReader[F] =
+    instance(_.readURL[F](8192))
+}

modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.bc
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+sealed trait AttachmentAction {}
+
+object AttachmentAction {
+
+  implicit val deriveConfig: Configuration =
+    Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames
+
+  case class SetExtractedText(text: Option[String]) extends AttachmentAction
+  object SetExtractedText {
+    implicit val jsonDecoder: Decoder[SetExtractedText] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetExtractedText] = deriveEncoder
+  }
+
+  implicit val jsonDecoder: Decoder[AttachmentAction] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[AttachmentAction] = deriveConfiguredEncoder
+
+}

modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.bc
+
+import docspell.common.Ident
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+sealed trait BackendCommand {}
+
+object BackendCommand {
+
+  implicit val deriveConfig: Configuration =
+    Configuration.default.withDiscriminator("command").withKebabCaseConstructorNames
+
+  case class ItemUpdate(itemId: Ident, actions: List[ItemAction]) extends BackendCommand
+  object ItemUpdate {
+    implicit val jsonDecoder: Decoder[ItemUpdate] = deriveDecoder
+    implicit val jsonEncoder: Encoder[ItemUpdate] = deriveEncoder
+  }
+
+  def item(itemId: Ident, actions: List[ItemAction]): BackendCommand =
+    ItemUpdate(itemId, actions)
+
+  case class AttachmentUpdate(
+      itemId: Ident,
+      attachId: Ident,
+      actions: List[AttachmentAction]
+  ) extends BackendCommand
+  object AttachmentUpdate {
+    implicit val jsonDecoder: Decoder[AttachmentUpdate] = deriveDecoder
+    implicit val jsonEncoder: Encoder[AttachmentUpdate] = deriveEncoder
+  }
+
+  implicit val jsonDecoder: Decoder[BackendCommand] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[BackendCommand] = deriveConfiguredEncoder
+}

modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.bc
+
+import docspell.common.Ident
+
+trait BackendCommandRunner[F[_], A] {
+
+  def run(collective: Ident, cmd: BackendCommand): F[A]
+
+  def runAll(collective: Ident, cmds: List[BackendCommand]): F[A]
+
+}

modules/common/src/main/scala/docspell/common/bc/ItemAction.scala

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.bc
+
+import docspell.common.Ident
+
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+sealed trait ItemAction {}
+
+object ItemAction {
+  implicit val deriveConfig: Configuration =
+    Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames
+
+  case class AddTags(tags: Set[String]) extends ItemAction
+  object AddTags {
+    implicit val jsonDecoder: Decoder[AddTags] = deriveDecoder
+    implicit val jsonEncoder: Encoder[AddTags] = deriveEncoder
+  }
+
+  case class ReplaceTags(tags: Set[String]) extends ItemAction
+  object ReplaceTags {
+    implicit val jsonDecoder: Decoder[ReplaceTags] = deriveDecoder
+    implicit val jsonEncoder: Encoder[ReplaceTags] = deriveEncoder
+  }
+
+  case class RemoveTags(tags: Set[String]) extends ItemAction
+  object RemoveTags {
+    implicit val jsonDecoder: Decoder[RemoveTags] = deriveDecoder
+    implicit val jsonEncoder: Encoder[RemoveTags] = deriveEncoder
+  }
+
+  case class RemoveTagsCategory(categories: Set[String]) extends ItemAction
+  object RemoveTagsCategory {
+    implicit val jsonDecoder: Decoder[RemoveTagsCategory] = deriveDecoder
+    implicit val jsonEncoder: Encoder[RemoveTagsCategory] = deriveEncoder
+  }
+
+  case class SetFolder(folder: Option[String]) extends ItemAction
+  object SetFolder {
+    implicit val jsonDecoder: Decoder[SetFolder] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetFolder] = deriveEncoder
+  }
+
+  case class SetCorrOrg(id: Option[Ident]) extends ItemAction
+  object SetCorrOrg {
+    implicit val jsonDecoder: Decoder[SetCorrOrg] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetCorrOrg] = deriveEncoder
+  }
+
+  case class SetCorrPerson(id: Option[Ident]) extends ItemAction
+  object SetCorrPerson {
+    implicit val jsonDecoder: Decoder[SetCorrPerson] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetCorrPerson] = deriveEncoder
+  }
+
+  case class SetConcPerson(id: Option[Ident]) extends ItemAction
+  object SetConcPerson {
+    implicit val jsonDecoder: Decoder[SetConcPerson] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetConcPerson] = deriveEncoder
+  }
+
+  case class SetConcEquipment(id: Option[Ident]) extends ItemAction
+  object SetConcEquipment {
+    implicit val jsonDecoder: Decoder[SetConcEquipment] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetConcEquipment] = deriveEncoder
+  }
+
+  case class SetField(field: Ident, value: String) extends ItemAction
+  object SetField {
+    implicit val jsonDecoder: Decoder[SetField] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetField] = deriveEncoder
+  }
+
+  case class SetName(name: String) extends ItemAction
+  object SetName {
+    implicit val jsonDecoder: Decoder[SetName] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetName] = deriveEncoder
+  }
+
+  case class SetNotes(notes: Option[String]) extends ItemAction
+  object SetNotes {
+    implicit val jsonDecoder: Decoder[SetNotes] = deriveDecoder
+    implicit val jsonEncoder: Encoder[SetNotes] = deriveEncoder
+  }
+
+  case class AddNotes(notes: String, separator: Option[String]) extends ItemAction
+  object AddNotes {
+    implicit val jsonDecoder: Decoder[AddNotes] = deriveDecoder
+    implicit val jsonEncoder: Encoder[AddNotes] = deriveEncoder
+  }
+
+  implicit val jsonDecoder: Decoder[ItemAction] = deriveConfiguredDecoder
+  implicit val jsonEncoder: Encoder[ItemAction] = deriveConfiguredEncoder
+}

modules/common/src/main/scala/docspell/common/exec/Args.scala

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+import fs2.io.file.Path
+
+case class Args(values: Seq[String]) {
+
+  def option(key: String, value: String): Args =
+    Args(values ++ Seq(key, value))
+
+  def option(key: String, value: Option[String]): Args =
+    value.map(v => option(key, v)).getOrElse(this)
+
+  def appendOpt(v: Option[String]): Args =
+    v.map(e => Args(values :+ e)).getOrElse(this)
+
+  def append(v: String, vs: String*): Args =
+    Args(values ++ (v +: vs))
+
+  def append(path: Path): Args =
+    append(path.toString)
+
+  def append(args: Args): Args =
+    Args(values ++ args.values)
+
+  def append(args: Seq[String]): Args =
+    Args(values ++ args)
+
+  def prepend(v: String): Args =
+    Args(v +: values)
+
+  def prependWhen(flag: Boolean)(v: String) =
+    if (flag) prepend(v) else this
+
+  def cmdString: String =
+    values.mkString(" ")
+}
+
+object Args {
+  val empty: Args = Args(Seq.empty)
+
+  def of(v: String*): Args =
+    Args(v)
+}

modules/common/src/main/scala/docspell/common/exec/Env.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+case class Env(values: Map[String, String]) {
+
+  def add(name: String, value: String): Env =
+    copy(values.updated(name, value))
+
+  def addAll(v: Map[String, String]): Env =
+    Env(values ++ v)
+
+  def addAll(e: Env): Env =
+    Env(values ++ e.values)
+
+  def ++(e: Env) = addAll(e)
+
+  def foreach(f: (String, String) => Unit): Unit =
+    values.foreach(t => f(t._1, t._2))
+
+  def map[A](f: (String, String) => A): Seq[A] =
+    values.map(f.tupled).toSeq
+
+  def mapConcat[A](f: (String, String) => Seq[A]): Seq[A] =
+    values.flatMap(f.tupled).toSeq
+}
+
+object Env {
+  val empty: Env = Env(Map.empty)
+
+  def of(nv: (String, String)*): Env =
+    Env(Map(nv: _*))
+}

modules/common/src/main/scala/docspell/common/exec/SysCmd.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+import docspell.common._
+
+final case class SysCmd(
+    program: String,
+    args: Args,
+    env: Env,
+    timeout: Duration
+) {
+
+  def withArgs(f: Args => Args): SysCmd =
+    copy(args = f(args))
+
+  def withTimeout(to: Duration): SysCmd =
+    copy(timeout = to)
+
+  def withEnv(f: Env => Env): SysCmd =
+    copy(env = f(env))
+
+  def addEnv(env: Env): SysCmd =
+    withEnv(_.addAll(env))
+
+  def cmdString: String =
+    s"$program ${args.cmdString}"
+
+  private[exec] def toCmd: Seq[String] =
+    program +: args.values
+}
+
+object SysCmd {
+  def apply(prg: String, args: String*): SysCmd =
+    apply(prg, Args(args))
+
+  def apply(prg: String, args: Args): SysCmd =
+    SysCmd(prg, args, Env.empty, Duration.minutes(2))
+}

modules/common/src/main/scala/docspell/common/exec/SysExec.scala

@@ -0,0 +1,163 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+import java.lang.ProcessBuilder.Redirect
+import java.util.concurrent.TimeUnit
+
+import scala.concurrent.TimeoutException
+import scala.jdk.CollectionConverters._
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.Path
+import fs2.{Pipe, Stream}
+
+import docspell.common.Duration
+import docspell.logging.Logger
+
+trait SysExec[F[_]] {
+
+  def stdout: Stream[F, Byte]
+
+  def stdoutLines: Stream[F, String] =
+    stdout
+      .through(fs2.text.utf8.decode)
+      .through(fs2.text.lines)
+
+  def stderr: Stream[F, Byte]
+
+  def stderrLines: Stream[F, String] =
+    stderr
+      .through(fs2.text.utf8.decode)
+      .through(fs2.text.lines)
+
+  def waitFor(timeout: Option[Duration] = None): F[Int]
+
+  /** Sends a signal to the process to terminate it immediately */
+  def cancel: F[Unit]
+
+  /** Consume lines of output of the process in background. */
+  def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit
+      F: Async[F]
+  ): Resource[F, SysExec[F]]
+
+  /** Consumes stderr lines (left) and stdout lines (right) in a background thread. */
+  def consumeOutputs(
+      m: Either[String, String] => F[Unit]
+  )(implicit F: Async[F]): Resource[F, SysExec[F]] = {
+    val pe: Pipe[F, String, Unit] = _.map(_.asLeft).evalMap(m)
+    val po: Pipe[F, String, Unit] = _.map(_.asRight).evalMap(m)
+    consumeOutputs(po, pe)
+  }
+
+  def logOutputs(logger: Logger[F], name: String)(implicit F: Async[F]) =
+    consumeOutputs {
+      case Right(line) => logger.debug(s"[$name (out)]: $line")
+      case Left(line)  => logger.debug(s"[$name (err)]: $line")
+    }
+}
+
+object SysExec {
+  private val readChunkSz = 8 * 1024
+
+  def apply[F[_]: Sync](
+      cmd: SysCmd,
+      logger: Logger[F],
+      workdir: Option[Path] = None,
+      stdin: Option[Stream[F, Byte]] = None
+  ): Resource[F, SysExec[F]] =
+    for {
+      proc <- startProcess(logger, cmd, workdir, stdin)
+      fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil))
+    } yield new SysExec[F] {
+      def stdout: Stream[F, Byte] =
+        fs2.io.readInputStream(
+          Sync[F].blocking(proc.getInputStream),
+          readChunkSz,
+          closeAfterUse = false
+        )
+
+      def stderr: Stream[F, Byte] =
+        fs2.io.readInputStream(
+          Sync[F].blocking(proc.getErrorStream),
+          readChunkSz,
+          closeAfterUse = false
+        )
+
+      def cancel = Sync[F].blocking(proc.destroy())
+
+      def waitFor(timeout: Option[Duration]): F[Int] = {
+        val to = timeout.getOrElse(cmd.timeout)
+        logger.trace("Waiting for command to terminate…") *>
+          Sync[F]
+            .blocking(proc.waitFor(to.millis, TimeUnit.MILLISECONDS))
+            .flatTap(_ => fibers.get.flatMap(_.traverse_(identity)))
+            .flatMap(terminated =>
+              if (terminated) proc.exitValue().pure[F]
+              else
+                Sync[F]
+                  .raiseError(
+                    new TimeoutException(s"Timed out after: ${to.formatExact}")
+                  )
+            )
+      }
+
+      def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit
+          F: Async[F]
+      ): Resource[F, SysExec[F]] =
+        for {
+          f1 <- F.background(stdoutLines.through(out).compile.drain)
+          f2 <- F.background(stderrLines.through(err).compile.drain)
+          _ <- Resource.eval(fibers.update(list => f1.void :: f2.void :: list))
+        } yield this
+    }
+
+  private def startProcess[F[_]: Sync, A](
+      logger: Logger[F],
+      cmd: SysCmd,
+      workdir: Option[Path],
+      stdin: Option[Stream[F, Byte]]
+  ): Resource[F, Process] = {
+    val log = logger.debug(s"Running external command: ${cmd.cmdString}")
+
+    val proc = log *>
+      Sync[F].blocking {
+        val pb = new ProcessBuilder(cmd.toCmd.asJava)
+          .redirectInput(if (stdin.isDefined) Redirect.PIPE else Redirect.INHERIT)
+          .redirectError(Redirect.PIPE)
+          .redirectOutput(Redirect.PIPE)
+
+        val pbEnv = pb.environment()
+        cmd.env.foreach { (name, v) =>
+          pbEnv.put(name, v)
+          ()
+        }
+        workdir.map(_.toNioPath.toFile).foreach(pb.directory)
+        pb.start()
+      }
+
+    Resource
+      .make(proc)(p =>
+        logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
+      )
+      .evalMap(p =>
+        stdin match {
+          case Some(in) =>
+            writeToProcess(in, p).compile.drain.as(p)
+          case None =>
+            p.pure[F]
+        }
+      )
+  }
+
+  private def writeToProcess[F[_]: Sync](
+      data: Stream[F, Byte],
+      proc: Process
+  ): Stream[F, Nothing] =
+    data.through(fs2.io.writeOutputStream(Sync[F].blocking(proc.getOutputStream)))
+}

modules/common/src/main/scala/docspell/common/util/File.scala

@@ -4,20 +4,18 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-package docspell.common
+package docspell.common.util
 
 import java.nio.file.{Path => JPath}
 
-import cats.FlatMap
-import cats.Monad
 import cats.effect._
-import cats.implicits._
+import cats.syntax.all._
+import cats.{FlatMap, Monad}
 import fs2.Stream
 import fs2.io.file.{Files, Flags, Path}
 
-import docspell.common.syntax.all._
-
 import io.circe.Decoder
+import io.circe.parser
 
 object File {
 
@@ -75,6 +73,5 @@ object File {
       .map(_ => file)
 
   def readJson[F[_]: Async, A](file: Path)(implicit d: Decoder[A]): F[A] =
-    readText[F](file).map(_.parseJsonAs[A]).rethrow
-
+    readText[F](file).map(parser.decode[A]).rethrow
 }

modules/common/src/main/scala/docspell/common/util/Random.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.util
+
+import cats.effect._
+
+import scodec.bits.ByteVector
+
+trait Random[F[_]] {
+  def string(len: Int): F[String]
+  def string: F[String] = string(8)
+}
+
+object Random {
+  def apply[F[_]: Sync] =
+    new Random[F] {
+      def string(len: Int) = Sync[F].delay {
+        val buf = Array.ofDim[Byte](len)
+        new scala.util.Random().nextBytes(buf)
+        ByteVector.view(buf).toBase58
+      }
+    }
+}

modules/common/src/test/scala/docspell/common/GlobTest.scala

@@ -70,11 +70,13 @@ class GlobTest extends FunSuite {
 
   test("with splitting") {
     assert(Glob("a/b/*").matches(true)("a/b/hello"))
+    assert(!Glob("a/b/*").matches(true)("a/b/hello/bello"))
     assert(!Glob("a/b/*").matches(true)("/a/b/hello"))
     assert(Glob("/a/b/*").matches(true)("/a/b/hello"))
     assert(!Glob("/a/b/*").matches(true)("a/b/hello"))
     assert(!Glob("*/a/b/*").matches(true)("a/b/hello"))
     assert(Glob("*/a/b/*").matches(true)("test/a/b/hello"))
+    assert(!Glob("/a/b").matches(true)("/a/b/c/d"))
   }
 
   test("asString") {

modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import munit._
+
+class UrlMatcherTest extends FunSuite {
+
+  test("it should match patterns") {
+    assertUrlsMatch(
+      uri("https://github.com/docspell/*") -> uri("https://github.com/docspell/dsc"),
+      uri("*s://test.com/*") -> uri("https://test.com/a"),
+      uri("*s://test.com/*") -> uri("https://test.com/a/b"),
+      uri("*s://test.com/*") -> uri("https://test.com/a/b/c"),
+      uri("*s://test.com/project/*") -> uri("https://test.com/project/c"),
+      uri("https://*.test.com/projects/*") -> uri("https://a.test.com/projects/p1"),
+      uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1"),
+      uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1")
+    )
+
+    assertUrlsNotMatch(
+      uri("https://*.test.com/projects/*") -> uri("https://test.com/projects/p1"),
+      uri("*s://test.com/project/*") -> uri("https://test.com/subject/c")
+    )
+  }
+
+  def uri(str: String): LenientUri = LenientUri.unsafe(str)
+
+  def assertUrlsMatch(tests: List[(LenientUri, LenientUri)]): Unit =
+    tests.foreach { case (patternUri, checkUri) =>
+      assert(
+        UrlMatcher.fromUrl(patternUri).matches(checkUri),
+        s"$patternUri does not match $checkUri"
+      )
+    }
+
+  def assertUrlsMatch(
+      test: (LenientUri, LenientUri),
+      more: (LenientUri, LenientUri)*
+  ): Unit =
+    assertUrlsMatch(test :: more.toList)
+
+  def assertUrlsNotMatch(tests: List[(LenientUri, LenientUri)]): Unit =
+    tests.foreach { case (patternUri, checkUri) =>
+      assert(
+        !UrlMatcher.fromUrl(patternUri).matches(checkUri),
+        s"$patternUri incorrectly matches $checkUri"
+      )
+    }
+
+  def assertUrlsNotMatch(
+      test: (LenientUri, LenientUri),
+      more: (LenientUri, LenientUri)*
+  ): Unit =
+    assertUrlsNotMatch(test :: more.toList)
+}

modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.bc
+
+import docspell.common._
+
+import io.circe.parser
+import io.circe.syntax._
+import munit._
+
+class BackendCommandTest extends FunSuite {
+
+  test("encode json") {
+    val bc: BackendCommand =
+      BackendCommand.item(
+        id("abc"),
+        List(
+          ItemAction.RemoveTagsCategory(Set("doctype")),
+          ItemAction.AddTags(Set("tag1", "tag2"))
+        )
+      )
+
+    assertEquals(
+      bc.asJson.spaces2,
+      """{
+        |  "itemId" : "abc",
+        |  "actions" : [
+        |    {
+        |      "categories" : [
+        |        "doctype"
+        |      ],
+        |      "action" : "remove-tags-category"
+        |    },
+        |    {
+        |      "tags" : [
+        |        "tag1",
+        |        "tag2"
+        |      ],
+        |      "action" : "add-tags"
+        |    }
+        |  ],
+        |  "command" : "item-update"
+        |}""".stripMargin
+    )
+  }
+
+  test("decode case insensitive keys") {
+    val json = """{
+                 |  "itemId" : "abc",
+                 |  "actions" : [
+                 |    {
+                 |      "categories" : [
+                 |        "doctype"
+                 |      ],
+                 |      "action" : "remove-tags-category"
+                 |    },
+                 |    {
+                 |      "tags" : [
+                 |        "tag1",
+                 |        "tag2"
+                 |      ],
+                 |      "action" : "add-tags"
+                 |    }
+                 |  ],
+                 |  "command" : "item-update"
+                 |}""".stripMargin
+
+    val bc: BackendCommand =
+      BackendCommand.item(
+        id("abc"),
+        List(
+          ItemAction.RemoveTagsCategory(Set("doctype")),
+          ItemAction.AddTags(Set("tag1", "tag2"))
+        )
+      )
+
+    assertEquals(parser.decode[BackendCommand](json), Right(bc))
+  }
+
+  def id(str: String) = Ident.unsafe(str)
+}

modules/config/src/main/scala/docspell/config/Implicits.scala

@@ -13,6 +13,7 @@ import scala.reflect.ClassTag
 import cats.syntax.all._
 import fs2.io.file.Path
 
+import docspell.addons.RunnerType
 import docspell.common._
 import docspell.ftspsql.{PgQueryParser, RankNormalization}
 import docspell.logging.{Level, LogConfig}
@@ -32,6 +33,17 @@ object Implicits {
         else super.fieldValue(name)
     }
 
+  implicit val urlMatcherReader: ConfigReader[UrlMatcher] = {
+    val fromList = ConfigReader[List[String]].emap(reason(UrlMatcher.fromStringList))
+    val fromString = ConfigReader[String].emap(
+      reason(str => UrlMatcher.fromStringList(str.split("[\\s,]+").toList))
+    )
+    fromList.orElse(fromString)
+  }
+
+  implicit val runnerSelectReader: ConfigReader[List[RunnerType]] =
+    ConfigReader[String].emap(reason(RunnerType.fromSeparatedString))
+
   implicit val accountIdReader: ConfigReader[AccountId] =
     ConfigReader[String].emap(reason(AccountId.parse))
 

modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala

@@ -12,6 +12,7 @@ import fs2.io.file.{Files, Path}
 import fs2.{Pipe, Stream}
 
 import docspell.common._
+import docspell.common.util.File
 import docspell.convert.ConversionResult
 import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt}
 import docspell.logging.Logger

modules/convert/src/test/scala/docspell/convert/ConversionTest.scala

@@ -15,6 +15,7 @@ import cats.implicits._
 import fs2.Stream
 
 import docspell.common._
+import docspell.common.util.File
 import docspell.convert.ConversionResult.Handler
 import docspell.convert.extern.OcrMyPdfConfig
 import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}

modules/convert/src/test/scala/docspell/convert/FileChecks.scala

@@ -18,6 +18,7 @@ import fs2.io.file.Path
 import fs2.{Pipe, Stream}
 
 import docspell.common._
+import docspell.common.util.File
 import docspell.convert.ConversionResult.Handler
 import docspell.files.TikaMimetype
 

modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala

@@ -14,6 +14,7 @@ import cats.effect.unsafe.implicits.global
 import fs2.io.file.Path
 
 import docspell.common._
+import docspell.common.util.File
 import docspell.convert._
 import docspell.files.ExampleFiles
 import docspell.logging.TestLoggingConfig

modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala

@@ -11,6 +11,7 @@ import fs2.Stream
 import fs2.io.file.Path
 
 import docspell.common._
+import docspell.common.util.File
 import docspell.logging.Logger
 
 object Ocr {

modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala

@@ -11,6 +11,7 @@ import java.nio.file.Paths
 import fs2.io.file.Path
 
 import docspell.common._
+import docspell.common.util.File
 
 case class OcrConfig(
     maxImageSize: Int,

modules/files/src/main/scala/docspell/files/FileSupport.scala

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.files
+
+import cats.data.OptionT
+import cats.effect.Sync
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common.{MimeType, MimeTypeHint}
+
+import io.circe.Encoder
+import io.circe.syntax._
+
+trait FileSupport {
+  implicit final class FileOps[F[_]: Files: Sync](self: Path) {
+    def detectMime: F[Option[MimeType]] =
+      Files[F].isReadable(self).flatMap { flag =>
+        OptionT
+          .whenF(flag) {
+            TikaMimetype
+              .detect(
+                Files[F].readAll(self),
+                MimeTypeHint.filename(self.fileName.toString)
+              )
+          }
+          .value
+      }
+
+    def asTextFile(alt: MimeType => F[Unit]): F[Option[Path]] =
+      OptionT(detectMime).flatMapF { mime =>
+        if (mime.matches(MimeType.text("plain"))) self.some.pure[F]
+        else alt(mime).as(None: Option[Path])
+      }.value
+
+    def readText: F[String] =
+      Files[F]
+        .readAll(self)
+        .through(fs2.text.utf8.decode)
+        .compile
+        .string
+
+    def readAll: Stream[F, Byte] =
+      Files[F].readAll(self)
+
+    def writeJson[A: Encoder](value: A): F[Unit] =
+      Stream
+        .emit(value.asJson.noSpaces)
+        .through(fs2.text.utf8.encode)
+        .through(Files[F].writeAll(self))
+        .compile
+        .drain
+  }
+}
+
+object FileSupport extends FileSupport

modules/files/src/main/scala/docspell/files/Zip.scala

@@ -8,11 +8,12 @@ package docspell.files
 
 import java.io.InputStream
 import java.nio.charset.StandardCharsets
-import java.nio.file.Paths
 import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
 
+import cats.data.OptionT
 import cats.effect._
 import cats.implicits._
+import fs2.io.file.{Files, Path}
 import fs2.{Pipe, Stream}
 
 import docspell.common.Binary
@@ -27,16 +28,72 @@ object Zip {
   ): Pipe[F, (String, Stream[F, Byte]), Byte] =
     in => zipJava(logger, chunkSize, in.through(deduplicate))
 
-  def unzipP[F[_]: Async](chunkSize: Int, glob: Glob): Pipe[F, Byte, Binary[F]] =
-    s => unzip[F](chunkSize, glob)(s)
+  def unzip[F[_]: Async](
+      chunkSize: Int,
+      glob: Glob
+  ): Pipe[F, Byte, Binary[F]] =
+    s => unzipStream[F](chunkSize, glob)(s)
 
-  def unzip[F[_]: Async](chunkSize: Int, glob: Glob)(
+  def unzipStream[F[_]: Async](chunkSize: Int, glob: Glob)(
       data: Stream[F, Byte]
   ): Stream[F, Binary[F]] =
     data
       .through(fs2.io.toInputStream[F])
       .flatMap(in => unzipJava(in, chunkSize, glob))
 
+  def saveTo[F[_]: Async](
+      logger: Logger[F],
+      targetDir: Path,
+      moveUp: Boolean
+  ): Pipe[F, Binary[F], Path] =
+    binaries =>
+      binaries
+        .filter(e => !e.name.endsWith("/"))
+        .evalMap { entry =>
+          val out = targetDir / entry.name
+          val createParent =
+            OptionT
+              .fromOption[F](out.parent)
+              .flatMapF(parent =>
+                Files[F]
+                  .exists(parent)
+                  .map(flag => Option.when(!flag)(parent))
+              )
+              .semiflatMap(p => Files[F].createDirectories(p))
+              .getOrElse(())
+
+          logger.trace(s"Unzip ${entry.name} -> $out") *>
+            createParent *>
+            entry.data.through(Files[F].writeAll(out)).compile.drain
+        }
+        .drain ++ Stream
+        .eval(if (moveUp) moveContentsUp(logger)(targetDir) else ().pure[F])
+        .as(targetDir)
+
+  private def moveContentsUp[F[_]: Sync: Files](logger: Logger[F])(dir: Path): F[Unit] =
+    Files[F]
+      .list(dir)
+      .take(2)
+      .compile
+      .toList
+      .flatMap {
+        case subdir :: Nil =>
+          Files[F].isDirectory(subdir).flatMap {
+            case false => ().pure[F]
+            case true =>
+              Files[F]
+                .list(subdir)
+                .filter(p => p != dir)
+                .evalTap(c => logger.trace(s"Move $c -> ${dir / c.fileName}"))
+                .evalMap(child => Files[F].move(child, dir / child.fileName))
+                .compile
+                .drain
+          }
+
+        case _ =>
+          ().pure[F]
+      }
+
   def unzipJava[F[_]: Async](
       in: InputStream,
       chunkSize: Int,
@@ -55,7 +112,7 @@ object Zip {
       .unNoneTerminate
       .filter(ze => glob.matchFilenameOrPath(ze.getName()))
       .map { ze =>
-        val name = Paths.get(ze.getName()).getFileName.toString
+        val name = ze.getName()
         val data =
           fs2.io.readInputStream[F]((zin: InputStream).pure[F], chunkSize, false)
         Binary(name, data)

modules/files/src/test/scala/docspell/files/ZipTest.scala

@@ -7,20 +7,25 @@
 package docspell.files
 
 import cats.effect._
-import cats.effect.unsafe.implicits.global
 import cats.implicits._
+import fs2.io.file.{Files, Path}
 
 import docspell.common.Glob
+import docspell.logging.TestLoggingConfig
 
 import munit._
 
-class ZipTest extends FunSuite {
+class ZipTest extends CatsEffectSuite with TestLoggingConfig {
+  val logger = docspell.logging.getLogger[IO]
+  val tempDir = ResourceFixture(
+    Files[IO].tempDirectory(Path("target").some, "zip-test-", None)
+  )
 
   test("unzip") {
     val zipFile = ExampleFiles.letters_zip.readURL[IO](8192)
-    val uncomp = zipFile.through(Zip.unzip(8192, Glob.all))
+    val unzip = zipFile.through(Zip.unzip(8192, Glob.all))
 
-    uncomp
+    unzip
       .evalMap { entry =>
         val x = entry.data.map(_ => 1).foldMonoid.compile.lastOrError
         x.map { size =>
@@ -35,6 +40,10 @@ class ZipTest extends FunSuite {
       }
       .compile
       .drain
-      .unsafeRunSync()
+  }
+
+  tempDir.test("unzipTo directory tree") { _ =>
+    // val zipFile = ExampleFiles.zip_dirs_zip.readURL[IO](8192)
+    // zipFile.through(Zip.unzip(G))
   }
 }

modules/joex/src/main/resources/reference.conf

@@ -780,4 +780,75 @@ Docpell Update Check
       index-all-chunk = 10
     }
   }
+
+  addons {
+    # A directory to extract addons when running them. Everything in
+    # here will be cleared after each run.
+    working-dir = ${java.io.tmpdir}"/docspell-addons"
+
+    # A directory for addons to store data between runs. This is not
+    # cleared by Docspell and can get large depending on the addons
+    # executed.
+    #
+    # This directory is used as base. In it subdirectories are created
+    # per run configuration id.
+    cache-dir = ${java.io.tmpdir}"/docspell-addon-cache"
+
+    executor-config {
+      # Define a (comma or whitespace separated) list of runners that
+      # are responsible for executing an addon. This setting is
+      # compared to what is supported by addons. Possible values are:
+      #
+      # - nix-flake: use nix-flake runner if the addon supports it
+      #   (this requires the nix package manager on the joex machine)
+      # - docker: use docker
+      # - trivial: use the trivial runner
+      #
+      # The first successful execution is used. This should list all
+      # runners the computer supports.
+      runner = "nix-flake, docker, trivial"
+
+      # systemd-nspawn can be used to run the program in a container.
+      # This is used by runners nix-flake and trivial.
+      nspawn = {
+        # If this is false, systemd-nspawn is not tried. When true, the
+        # addon is executed inside a lightweight container via
+        # systemd-nspawn.
+        enabled = false
+
+        # Path to sudo command. By default systemd-nspawn is executed
+        # via sudo - the user running joex must be allowed to do so NON
+        # INTERACTIVELY. If this is empty, then nspawn is tried to
+        # execute without sudo.
+        sudo-binary = "sudo"
+
+        # Path to the systemd-nspawn command.
+        nspawn-binary = "systemd-nspawn"
+
+        # Workaround, if multiple same named containers are run too fast
+        container-wait = "100 millis"
+      }
+
+      # The timeout for running an addon.
+      run-timeout = "15 minutes"
+
+      # Configure the nix flake runner.
+      nix-runner {
+        # Path to the nix command.
+        nix-binary = "nix"
+
+        # The timeout for building the package (running nix build).
+        build-timeout = "15 minutes"
+      }
+
+      # Configure the docker runner
+      docker-runner {
+        # Path to the docker command.
+        docker-binary = "docker"
+
+        # The timeout for building the package (running docker build).
+        build-timeout = "15 minutes"
+      }
+    }
+  }
 }

modules/joex/src/main/scala/docspell/joex/Config.scala

@@ -12,6 +12,7 @@ import fs2.io.file.Path
 import docspell.analysis.TextAnalysisConfig
 import docspell.analysis.classifier.TextClassifierConfig
 import docspell.backend.Config.Files
+import docspell.backend.joex.AddonEnvConfig
 import docspell.common._
 import docspell.config.{FtsType, PgFtsConfig}
 import docspell.convert.ConvertConfig
@@ -43,7 +44,8 @@ case class Config(
     files: Files,
     mailDebug: Boolean,
     fullTextSearch: Config.FullTextSearch,
-    updateCheck: UpdateCheckConfig
+    updateCheck: UpdateCheckConfig,
+    addons: AddonEnvConfig
 ) {
 
   def pubSubConfig(headerValue: Ident): PubSubConfig =

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -145,6 +145,8 @@ object JoexAppImpl extends MailAddressCodec {
         schedulerModule.scheduler,
         schedulerModule.periodicScheduler
       )
+      nodes <- ONode(store)
+      _ <- nodes.withRegistered(cfg.appId, NodeType.Joex, cfg.baseUrl, None)
       appR <- Resource.make(app.init.map(_ => app))(_.initShutdown)
     } yield appR
 

modules/joex/src/main/scala/docspell/joex/JoexServer.scala

@@ -59,7 +59,7 @@ object JoexServer {
           Router("pubsub" -> pubSub.receiveRoute)
         },
         "/api/info" -> InfoRoutes(cfg),
-        "/api/v1" -> JoexRoutes(joexApp)
+        "/api/v1" -> JoexRoutes(cfg, joexApp)
       ).orNotFound
 
       // With Middlewares in place