diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala index f13af4ef..68715cf3 100644 --- a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala @@ -46,7 +46,12 @@ object AddonExecutor { in.cacheDir, in.addons ) - rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c)) + rs <- + if (cfg.failFast) ctx.foldLeftM(List.empty[AddonResult]) { (res, c) => + if (res.headOption.exists(_.isFailure)) res.pure[F] + else runAddon(logger.withAddon(c), in.env)(c).map(r => r :: res) + } + else ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c)) pure = ctx.foldl(true)((b, c) => b && c.meta.isPure) } yield AddonExecutionResult(rs, pure) } diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala index b4c29cb1..e4f3c520 100644 --- a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala @@ -15,7 +15,8 @@ case class AddonExecutorConfig( runTimeout: Duration, nspawn: NSpawn, nixRunner: NixConfig, - dockerRunner: DockerConfig + dockerRunner: DockerConfig, + failFast: Boolean ) object AddonExecutorConfig { diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala index 7efa9c26..c4e9f5ae 100644 --- a/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala @@ -25,6 +25,7 @@ sealed trait AddonResult { } object AddonResult { + val emptySuccess: AddonResult = success(AddonOutput.empty) /** The addon was run successful, but decoding its stdout failed. */ case class DecodingError(message: String) extends AddonResult { diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala index 5ad59b14..946befab 100644 --- a/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala @@ -7,7 +7,11 @@ package docspell.addons import cats.effect._ +import cats.syntax.all._ +import docspell.addons.out.AddonOutput +import docspell.common.UrlReader +import docspell.common.bc.{BackendCommand, ItemAction} import docspell.logging.{Level, TestLoggingConfig} import munit._ @@ -60,4 +64,76 @@ class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingCo _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake)) } yield () } + + tempDir.test("fail early if configured so") { dir => + val cfg = testExecutorConfig(RunnerType.Trivial).copy(failFast = true) + val exec = AddonExecutor[IO](cfg, UrlReader.defaultReader).execute(logger) + val testOut = AddonOutput(commands = + List( + BackendCommand.item(id("xyz-item"), List(ItemAction.AddTags(Set("tag1", "tag2")))) + ) + ) + val result = createInputEnv( + dir, + AddonGenerator.failingAddon("addon1"), + AddonGenerator.successAddon("addon2", output = testOut.some) + ).use(exec.run) + result.map { res => + assert(res.isFailure) + assert(res.pure) + assertEquals(res.addonResult, AddonResult.executionError(1)) + assertEquals(res.addonResults.size, 1) + } + } + + tempDir.test("do not stop after failing addons") { dir => + val cfg = testExecutorConfig(RunnerType.Trivial).copy(failFast = false) + val exec = AddonExecutor[IO](cfg, UrlReader.defaultReader).execute(logger) + val testOut = AddonOutput(commands = + List( + BackendCommand.item(id("xyz-item"), List(ItemAction.AddTags(Set("tag1", "tag2")))) + ) + ) + val result = createInputEnv( + dir, + AddonGenerator.failingAddon("addon1"), + AddonGenerator.successAddon("addon2", output = testOut.some) + ).use(exec.run) + result.map { res => + assert(res.isFailure) + assert(res.pure) + assertEquals(res.addonResult, AddonResult.executionError(1)) + assertEquals(res.addonResults.size, 2) + assertEquals(res.addonResults.head, AddonResult.executionError(1)) + assertEquals(res.addonResults(1), AddonResult.success(testOut)) + } + } + + tempDir.test("combine outputs") { dir => + val cfg = testExecutorConfig(RunnerType.Trivial).copy(failFast = false) + val exec = AddonExecutor[IO](cfg, UrlReader.defaultReader).execute(logger) + val testOut1 = AddonOutput(commands = + List( + BackendCommand.item(id("xyz-item"), List(ItemAction.AddTags(Set("tag1", "tag2")))) + ) + ) + val testOut2 = AddonOutput(commands = + List( + BackendCommand.item(id("xyz-item"), List(ItemAction.SetName("new item name"))) + ) + ) + val result = createInputEnv( + dir, + AddonGenerator.successAddon("addon1", output = testOut1.some), + AddonGenerator.successAddon("addon2", output = testOut2.some) + ).use(exec.run) + result.map { res => + assert(res.isSuccess) + assert(res.pure) + assertEquals(res.addonResult, AddonResult.success(testOut1.combine(testOut2))) + assertEquals(res.addonResults.size, 2) + assertEquals(res.addonResults.head, AddonResult.success(testOut1)) + assertEquals(res.addonResults(1), AddonResult.success(testOut2)) + } + } } diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala new file mode 100644 index 00000000..11a546b9 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala @@ -0,0 +1,115 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect.{IO, Resource} +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path, PosixPermissions} + +import docspell.addons.out.AddonOutput +import docspell.common.LenientUri +import docspell.files.Zip + +import io.circe.syntax._ + +object AddonGenerator { + private[this] val logger = docspell.logging.getLogger[IO] + + def successAddon( + name: String, + version: String = "1.0", + output: Option[AddonOutput] = None + ): Resource[IO, AddonArchive] = + output match { + case None => + generate(name, version, false)("exit 0") + case Some(out) => + generate(name, version, true)( + s""" + |cat <<-EOF + |${out.asJson.noSpaces} + |EOF""".stripMargin + ) + } + + def failingAddon( + name: String, + version: String = "1.0", + pure: Boolean = true + ): Resource[IO, AddonArchive] = + generate(name, version, pure)("exit 1") + + def generate(name: String, version: String, collectOutput: Boolean)( + script: String + ): Resource[IO, AddonArchive] = + Files[IO].tempDirectory(None, s"addon-gen-$name-$version-", None).evalMap { dir => + for { + yml <- createDescriptor(dir, name, version, collectOutput) + bin <- createScript(dir, script) + zip <- createZip(dir, List(yml, bin)) + url = LenientUri.fromJava(zip.toNioPath.toUri.toURL) + } yield AddonArchive(url, name, version) + } + + private def createZip(dir: Path, files: List[Path]) = + Stream + .emits(files) + .map(f => (f.fileName.toString, Files[IO].readAll(f))) + .covary[IO] + .through(Zip.zip[IO](logger, 8192)) + .through(Files[IO].writeAll(dir / "addon.zip")) + .compile + .drain + .as(dir / "addon.zip") + + private def createDescriptor( + dir: Path, + name: String, + version: String, + collectOutput: Boolean + ): IO[Path] = { + val meta = AddonMeta( + meta = AddonMeta.Meta(name, version, None), + triggers = Set(AddonTriggerType.ExistingItem: AddonTriggerType).some, + args = None, + runner = + AddonMeta.Runner(None, None, AddonMeta.TrivialRunner(true, "addon.sh").some).some, + options = + AddonMeta.Options(networking = !collectOutput, collectOutput = collectOutput).some + ) + + Stream + .emit(meta.asJson.noSpaces) + .covary[IO] + .through(fs2.text.utf8.encode) + .through(Files[IO].writeAll(dir / "docspell-addon.json")) + .compile + .drain + .as(dir / "docspell-addon.json") + } + + private def createScript(dir: Path, content: String): IO[Path] = { + val scriptFile = dir / "addon.sh" + Stream + .emit(s""" + |#!/usr/bin/env bash + | + |$content + | + |""".stripMargin) + .covary[IO] + .through(fs2.text.utf8.encode) + .through(Files[IO].writeAll(scriptFile)) + .compile + .drain + .as(scriptFile) + .flatTap(f => + Files[IO].setPosixPermissions(f, PosixPermissions.fromOctal("777").get) + ) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala index c20ac112..20abfa95 100644 --- a/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala +++ b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala @@ -13,7 +13,8 @@ import fs2.io.file.{Files, Path, PosixPermissions} import docspell.addons.AddonExecutorConfig._ import docspell.addons.AddonMeta._ import docspell.addons.AddonTriggerType._ -import docspell.common.{Duration, LenientUri} +import docspell.common.exec.Env +import docspell.common.{Duration, Ident, LenientUri} import docspell.logging.TestLoggingConfig import munit.CatsEffectSuite @@ -22,6 +23,8 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite => val files: Files[IO] = Files[IO] + def id(str: String): Ident = Ident.unsafe(str) + val dummyAddonUrl = LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip")) @@ -59,13 +62,24 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite => runner: RunnerType, runners: RunnerType* ): AddonExecutorConfig = { - val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100)) + val nspawn = NSpawn(false, "sudo", "systemd-nspawn", Duration.millis(100)) AddonExecutorConfig( - runner :: runners.toList, - Duration.minutes(2), - nspawn, - NixConfig("nix", Duration.minutes(2)), - DockerConfig("docker", Duration.minutes(2)) + runner = runner :: runners.toList, + runTimeout = Duration.minutes(2), + nspawn = nspawn, + nixRunner = NixConfig("nix", Duration.minutes(2)), + dockerRunner = DockerConfig("docker", Duration.minutes(2)), + failFast = true ) } + + def createInputEnv( + dir: Path, + addon: Resource[IO, AddonArchive], + more: Resource[IO, AddonArchive]* + ): Resource[IO, InputEnv] = + (addon :: more.toList) + .traverse(_.map(a => AddonRef(a, ""))) + .map(addons => InputEnv(addons, dir, dir, dir, Env.empty)) + } diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 7a7d9b97..a0c9e9b7 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -829,6 +829,11 @@ Docpell Update Check container-wait = "100 millis" } + # When multiple addons are executed sequentially, stop after the + # first failing result. If this is false, then subsequent addons + # will be run for their side effects only. + fail-fast = true + # The timeout for running an addon. run-timeout = "15 minutes"