Experiment with addons

Addons allow to execute external programs in some context inside
docspell. Currently it is possible to run them after processing files.
Addons are provided by URLs to zip files.

modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.option._
+
+import docspell.common.UrlReader
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  tempDir.test("Read archive from directory") { dir =>
+    for {
+      archive <- IO(AddonArchive(dummyAddonUrl, "", ""))
+      path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir)
+
+      aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some)
+      _ = {
+        assertEquals(aa.name, "dummy-addon")
+        assertEquals(aa.version, "2.9")
+        assertEquals(aa.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+
+  test("Read archive from zip file") {
+    for {
+      archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO])
+      _ = {
+        assertEquals(archive.name, "dummy-addon")
+        assertEquals(archive.version, "2.9")
+        assertEquals(archive.url, dummyAddonUrl)
+      }
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.logging.{Level, TestLoggingConfig}
+
+import munit._
+
+class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig {
+  val logger = docspell.logging.getLogger[IO]
+
+  override def docspellLogConfig =
+    super.docspellLogConfig.copy(minimumLevel = Level.Trace)
+
+  tempDir.test("select docker if Dockerfile exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake if flake.nix exists") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.NixFlake))
+    } yield ()
+  }
+
+  tempDir.test("select nix-flake and docker") { dir =>
+    for {
+      _ <- files.createFile(dir / "flake.nix")
+      _ <- files.createFile(dir / "Dockerfile")
+      cfg = testExecutorConfig(
+        RunnerType.Docker,
+        RunnerType.NixFlake,
+        RunnerType.Trivial
+      )
+      meta = dummyAddonMeta.copy(runner = None)
+      r <- AddonExecutor.selectRunner[IO](cfg, meta, dir)
+      _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake))
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+
+import docspell.common.Glob
+import docspell.files.Zip
+import docspell.logging.TestLoggingConfig
+
+import munit._
+
+class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures {
+  val logger = docspell.logging.getLogger[IO]
+
+  test("read meta from zip file") {
+    val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192))
+    assertIO(meta, dummyAddonMeta)
+  }
+
+  tempDir.test("read meta from directory") { dir =>
+    for {
+      _ <- dummyAddonUrl
+        .readURL[IO](8192)
+        .through(Zip.unzip(8192, Glob.all))
+        .through(Zip.saveTo(logger, dir, moveUp = true))
+        .compile
+        .drain
+      meta <- AddonMeta.findInDirectory[IO](dir)
+      _ = assertEquals(meta, dummyAddonMeta)
+    } yield ()
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import docspell.addons.out.AddonOutput
+
+import io.circe.parser.decode
+import munit.FunSuite
+
+class AddonOutputTest extends FunSuite {
+
+  test("decode empty object") {
+    val out = decode[AddonOutput]("{}")
+    println(out)
+  }
+
+  test("decode sample output") {
+    val jsonStr =
+      """{ "files": [
+        |    {
+        |      "itemId": "qZDnyGIAJsXr",
+        |      "textFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.txt"
+        |      },
+        |      "pdfFiles": {
+        |        "HPFvIDib6eA": "HPFvIDib6eA.pdf"
+        |      }
+        |    }
+        |  ]
+        |}
+        |""".stripMargin
+
+    val out = decode[AddonOutput](jsonStr)
+    println(out)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import java.util.concurrent.atomic.AtomicInteger
+
+import cats.effect.IO
+import fs2.io.file.Path
+
+import docspell.common.LenientUri
+import docspell.common.exec.Env
+import docspell.logging.{Logger, TestLoggingConfig}
+
+import munit._
+
+class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig {
+
+  val logger = docspell.logging.getLogger[IO]
+
+  val dummyContext = Context(
+    addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""),
+    meta = AddonMeta.empty("test", "1.0"),
+    baseDir = Path(""),
+    addonDir = Path(""),
+    outputDir = Path(""),
+    cacheDir = Path("")
+  )
+
+  test("firstSuccessful must stop on first success") {
+    val counter = new AtomicInteger(0)
+    val runner = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(runner, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("firstSuccessful must try with next on error") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.failWith[IO]("failed")
+    val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+    val r = AddonRunner.firstSuccessful(fail, runner, runner)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 1)
+    } yield ()
+  }
+
+  test("do not retry on decoding errors") {
+    val counter = new AtomicInteger(0)
+    val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(fail, increment, increment)
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 0)
+    } yield ()
+  }
+
+  test("try on errors but stop on decoding error") {
+    val counter = new AtomicInteger(0)
+    val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed"))
+    val incrementFail =
+      new MockRunner(IO(counter.incrementAndGet()).void)
+        .as(AddonResult.executionFailed(new Exception("fail")))
+    val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void)
+
+    val r = AddonRunner.firstSuccessful(
+      incrementFail,
+      incrementFail,
+      decodeFail,
+      increment,
+      increment
+    )
+    for {
+      _ <- r.run(logger, Env.empty, dummyContext)
+      _ = assertEquals(counter.get(), 2)
+    } yield ()
+  }
+
+  final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty)
+      extends AddonRunner[IO] {
+    val runnerType = Nil
+    def run(
+        logger: Logger[IO],
+        env: Env,
+        ctx: Context
+    ) = run.as(result)
+
+    def as(r: AddonResult) = new MockRunner(run, r)
+  }
+}

modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.io.file.{Files, Path, PosixPermissions}
+
+import docspell.addons.AddonExecutorConfig._
+import docspell.addons.AddonMeta._
+import docspell.addons.AddonTriggerType._
+import docspell.common.{Duration, LenientUri}
+import docspell.logging.TestLoggingConfig
+
+import munit.CatsEffectSuite
+
+trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
+
+  val files: Files[IO] = Files[IO]
+
+  val dummyAddonUrl =
+    LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip"))
+
+  val dummyAddonMeta =
+    AddonMeta(
+      meta =
+        AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some),
+      triggers = Some(
+        Set(Scheduled, FinalProcessItem, FinalReprocessItem)
+      ),
+      None,
+      runner = Runner(
+        nix = NixRunner(true).some,
+        docker = DockerRunner(
+          enable = true,
+          image = None,
+          build = "Dockerfile".some
+        ).some,
+        trivial = TrivialRunner(true, "src/addon.sh").some
+      ).some,
+      options = Options(networking = true, collectOutput = true).some
+    )
+
+  def baseTempDir: Path =
+    Path(s"/tmp/target/test-temp")
+
+  val tempDir =
+    ResourceFixture[Path](
+      Resource.eval(Files[IO].createDirectories(baseTempDir)) *>
+        Files[IO]
+          .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777"))
+    )
+
+  def testExecutorConfig(
+      runner: RunnerType,
+      runners: RunnerType*
+  ): AddonExecutorConfig = {
+    val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100))
+    AddonExecutorConfig(
+      runner :: runners.toList,
+      Duration.minutes(2),
+      nspawn,
+      NixConfig("nix", Duration.minutes(2)),
+      DockerConfig("docker", Duration.minutes(2))
+    )
+  }
+}