diff --git a/build.sbt b/build.sbt index 2240a4e0..8156fc49 100644 --- a/build.sbt +++ b/build.sbt @@ -293,6 +293,15 @@ val openapiScalaSettings = Seq( field.copy(typeDef = TypeDef("DownloadState", Imports("docspell.common.DownloadState")) ) + case "addon-trigger-type" => + field => + field.copy(typeDef = + TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType")) + ) + case "addon-runner-type" => + field => + field + .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType"))) }) ) @@ -325,6 +334,7 @@ val common = project libraryDependencies ++= Dependencies.fs2 ++ Dependencies.circe ++ + Dependencies.circeGenericExtra ++ Dependencies.calevCore ++ Dependencies.calevCirce ) @@ -351,7 +361,7 @@ val files = project .in(file("modules/files")) .disablePlugins(RevolverPlugin) .settings(sharedSettings) - .withTestSettings + .withTestSettingsDependsOn(loggingScribe) .settings( name := "docspell-files", libraryDependencies ++= @@ -448,6 +458,19 @@ val notificationApi = project ) .dependsOn(common, loggingScribe) +val addonlib = project + .in(file("modules/addonlib")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .withTestSettingsDependsOn(loggingScribe) + .settings( + libraryDependencies ++= + Dependencies.fs2 ++ + Dependencies.circe ++ + Dependencies.circeYaml + ) + .dependsOn(common, files, loggingScribe) + val store = project .in(file("modules/store")) .disablePlugins(RevolverPlugin) @@ -469,7 +492,16 @@ val store = project libraryDependencies ++= Dependencies.testContainer.map(_ % Test) ) - .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe) + .dependsOn( + common, + addonlib, + query.jvm, + totp, + files, + notificationApi, + jsonminiq, + loggingScribe + ) val notificationImpl = project .in(file("modules/notification/impl")) @@ -647,7 +679,7 @@ val restapi = project openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, query.jvm, notificationApi, jsonminiq) + .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib) val joexapi = project .in(file("modules/joexapi")) @@ -667,7 +699,7 @@ val joexapi = project openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, loggingScribe) + .dependsOn(common, loggingScribe, addonlib) val backend = project .in(file("modules/backend")) @@ -683,6 +715,7 @@ val backend = project Dependencies.emil ) .dependsOn( + addonlib, store, notificationApi, joexapi, @@ -739,7 +772,7 @@ val config = project Dependencies.fs2 ++ Dependencies.pureconfig ) - .dependsOn(common, loggingApi, ftspsql, store) + .dependsOn(common, loggingApi, ftspsql, store, addonlib) // --- Application(s) @@ -946,6 +979,7 @@ val root = project ) .aggregate( common, + addonlib, loggingApi, loggingScribe, config, diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala new file mode 100644 index 00000000..3c2d6051 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.Zip + +final case class AddonArchive(url: LenientUri, name: String, version: String) { + def nameAndVersion: String = + s"$name-$version" + + def extractTo[F[_]: Async]( + reader: UrlReader[F], + directory: Path, + withSubdir: Boolean = true, + glob: Glob = Glob.all + ): F[Path] = { + val logger = docspell.logging.getLogger[F] + val target = + if (withSubdir) directory.absolute / nameAndVersion + else directory.absolute + + Files[F] + .exists(target) + .flatMap { + case true => target.pure[F] + case false => + Files[F].createDirectories(target) *> + reader(url) + .through(Zip.unzip(8192, glob)) + .through(Zip.saveTo(logger, target, moveUp = true)) + .compile + .drain + .as(target) + } + } + + /** Read meta either from the given directory or extract the url to find the metadata + * file to read + */ + def readMeta[F[_]: Async]( + urlReader: UrlReader[F], + directory: Option[Path] = None + ): F[AddonMeta] = + directory + .map(AddonMeta.findInDirectory[F]) + .getOrElse(AddonMeta.findInZip(urlReader(url))) +} + +object AddonArchive { + def read[F[_]: Async]( + url: LenientUri, + urlReader: UrlReader[F], + extractDir: Option[Path] = None + ): F[AddonArchive] = { + val addon = AddonArchive(url, "", "") + addon + .readMeta(urlReader, extractDir) + .map(m => addon.copy(name = m.meta.name, version = m.meta.version)) + } + + def dockerAndFlakeExists[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[(Boolean, Boolean)] = { + val files = Files[F] + def forPath(path: Path): F[(Boolean, Boolean)] = + (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled + + def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] = + data + .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix"))) + .collect { + case bin if bin.name == "Dockerfile" => (true, false) + case bin if bin.name == "flake.nix" => (false, true) + } + .compile + .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2)) + + archive.fold(forPath, forZip) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala new file mode 100644 index 00000000..08b3fd7b --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid +import cats.syntax.all._ + +case class AddonExecutionResult( + addonResults: List[AddonResult], + pure: Boolean +) { + def addonResult: AddonResult = addonResults.combineAll + def isFailure: Boolean = addonResult.isFailure + def isSuccess: Boolean = addonResult.isSuccess +} + +object AddonExecutionResult { + val empty: AddonExecutionResult = + AddonExecutionResult(Nil, false) + + def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult = + AddonExecutionResult( + a.addonResults ::: b.addonResults, + a.pure && b.pure + ) + + implicit val executionResultMonoid: Monoid[AddonExecutionResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala new file mode 100644 index 00000000..f13af4ef --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.Kleisli +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file._ + +import docspell.common.UrlReader +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonExecutor[F[_]] { + + def config: AddonExecutorConfig + + def execute(logger: Logger[F]): AddonExec[F] + + def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] = + execute(logger).run(in) +} + +object AddonExecutor { + + def apply[F[_]: Async]( + cfg: AddonExecutorConfig, + urlReader: UrlReader[F] + ): AddonExecutor[F] = + new AddonExecutor[F] with AddonLoggerExtension { + val config = cfg + + def execute(logger: Logger[F]): AddonExec[F] = + Kleisli { in => + for { + _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}") + ctx <- prepareDirectory( + logger, + in.baseDir, + in.outputDir, + in.cacheDir, + in.addons + ) + rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c)) + pure = ctx.foldl(true)((b, c) => b && c.meta.isPure) + } yield AddonExecutionResult(rs, pure) + } + + private def prepareDirectory( + logger: Logger[F], + baseDir: Path, + outDir: Path, + cacheDir: Path, + addons: List[AddonRef] + ): F[List[Context]] = + for { + addonsDir <- Directory.create(baseDir / "addons") + _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir) + _ <- Context + .userInputFile(baseDir) + .parent + .fold(().pure[F])(Files[F].createDirectories) + archives = addons.map(_.archive).distinctBy(_.url) + _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir") + mkCtxs <- archives.traverse { archive => + for { + _ <- logger.debug(s"Extracting $archive") + addonDir <- archive.extractTo(urlReader, addonsDir) + meta <- AddonMeta.findInDirectory(addonDir) + mkCtx = (ref: AddonRef) => + Context(ref, meta, baseDir, addonDir, outDir, cacheDir) + } yield archive.url -> mkCtx + } + ctxFactory = mkCtxs.toMap + res = addons.map(ref => ctxFactory(ref.archive.url)(ref)) + } yield res + + private def runAddon(logger: Logger[F], env: Env)( + ctx: Context + ): F[AddonResult] = + for { + _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}") + _ <- logger.trace("Storing user input into file") + _ <- Stream + .emit(ctx.addon.args) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(ctx.userInputFile, Flags.Write)) + .compile + .drain + + runner <- selectRunner(cfg, ctx.meta, ctx.addonDir) + result <- runner.run(logger, env, ctx) + } yield result + } + + def selectRunner[F[_]: Async]( + cfg: AddonExecutorConfig, + meta: AddonMeta, + addonDir: Path + ): F[AddonRunner[F]] = + for { + addonRunner <- meta.enabledTypes(Left(addonDir)) + // intersect on list retains order in first + possibleRunner = cfg.runner + .intersect(addonRunner) + .map(AddonRunner.forType[F](cfg)) + runner = possibleRunner match { + case Nil => + AddonRunner.failWith( + s"No runner available for addon config ${meta.runner} and config ${cfg.runner}." + ) + case list => + AddonRunner.firstSuccessful(list) + } + } yield runner +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala new file mode 100644 index 00000000..b4c29cb1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.AddonExecutorConfig._ +import docspell.common.Duration +import docspell.common.exec.{Args, SysCmd} + +case class AddonExecutorConfig( + runner: List[RunnerType], + runTimeout: Duration, + nspawn: NSpawn, + nixRunner: NixConfig, + dockerRunner: DockerConfig +) + +object AddonExecutorConfig { + + case class NSpawn( + enabled: Boolean, + sudoBinary: String, + nspawnBinary: String, + containerWait: Duration + ) { + val nspawnVersion = + SysCmd(nspawnBinary, Args.of("--version")).withTimeout(Duration.seconds(2)) + } + + case class NixConfig( + nixBinary: String, + buildTimeout: Duration + ) + + case class DockerConfig( + dockerBinary: String, + buildTimeout: Duration + ) { + def dockerBuild(imageName: String): SysCmd = + SysCmd(dockerBinary, "build", "-t", imageName, ".").withTimeout(buildTimeout) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala new file mode 100644 index 00000000..1b390985 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.logging.Logger + +trait AddonLoggerExtension { + + implicit final class LoggerAddonOps[F[_]](self: Logger[F]) { + private val addonName = "addon-name" + private val addonVersion = "addon-version" + + def withAddon(r: AddonArchive): Logger[F] = + self.capture(addonName, r.name).capture(addonVersion, r.version) + + def withAddon(r: Context): Logger[F] = + withAddon(r.addon.archive) + + def withAddon(m: AddonMeta): Logger[F] = + self.capture(addonName, m.meta.name).capture(addonVersion, m.meta.version) + } +} + +object AddonLoggerExtension extends AddonLoggerExtension diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala new file mode 100644 index 00000000..649b5d89 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.io.FileNotFoundException + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.Glob +import docspell.files.Zip + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.yaml.{parser => YamlParser} +import io.circe.{Decoder, Encoder} +import io.circe.{parser => JsonParser} + +case class AddonMeta( + meta: AddonMeta.Meta, + triggers: Option[Set[AddonTriggerType]], + args: Option[List[String]], + runner: Option[AddonMeta.Runner], + options: Option[AddonMeta.Options] +) { + + def nameAndVersion: String = + s"${meta.name}-${meta.version}" + + def parseResult: Boolean = + options.exists(_.collectOutput) + + def ignoreResult: Boolean = + !parseResult + + def isImpure: Boolean = + options.exists(_.isImpure) + + def isPure: Boolean = + options.forall(_.isPure) + + /** Returns a list of runner types that are possible to use for this addon. This is also + * inspecting the archive to return defaults when the addon isn't declaring it in the + * descriptor. + */ + def enabledTypes[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[List[RunnerType]] = + for { + filesExists <- AddonArchive.dockerAndFlakeExists(archive) + (dockerFileExists, flakeFileExists) = filesExists + + nixEnabled = runner.flatMap(_.nix).map(_.enable) match { + case Some(flag) => flag + case None => flakeFileExists + } + + dockerEnabled = runner.flatMap(_.docker).map(_.enable) match { + case Some(flag) => flag + case None => dockerFileExists + } + + trivialEnabled = runner.flatMap(_.trivial).exists(_.enable) + + result = RunnerType.all.filter(_.fold(nixEnabled, dockerEnabled, trivialEnabled)) + } yield result + +} + +object AddonMeta { + + def empty(name: String, version: String): AddonMeta = + AddonMeta(Meta(name, version, None), None, None, None, None) + + case class Meta(name: String, version: String, description: Option[String]) + case class Runner( + nix: Option[NixRunner], + docker: Option[DockerRunner], + trivial: Option[TrivialRunner] + ) + case class NixRunner(enable: Boolean) + case class DockerRunner(enable: Boolean, image: Option[String], build: Option[String]) + case class TrivialRunner(enable: Boolean, exec: String) + case class Options(networking: Boolean, collectOutput: Boolean) { + def isPure = !networking && collectOutput + def isImpure = networking + def isUseless = !networking && !collectOutput + def isUseful = networking || collectOutput + } + + object NixRunner { + implicit val jsonEncoder: Encoder[NixRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[NixRunner] = + deriveDecoder + } + + object DockerRunner { + implicit val jsonEncoder: Encoder[DockerRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[DockerRunner] = + deriveDecoder + } + + object TrivialRunner { + implicit val jsonEncoder: Encoder[TrivialRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[TrivialRunner] = + deriveDecoder + } + + object Runner { + implicit val jsonEncoder: Encoder[Runner] = + deriveEncoder + implicit val jsonDecoder: Decoder[Runner] = + deriveDecoder + } + + object Options { + implicit val jsonEncoder: Encoder[Options] = + deriveEncoder + implicit val jsonDecoder: Decoder[Options] = + deriveDecoder + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = + deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = + deriveDecoder + } + + implicit val jsonEncoder: Encoder[AddonMeta] = + deriveEncoder + + implicit val jsonDecoder: Decoder[AddonMeta] = + deriveDecoder + + def fromJsonString(str: String): Either[Throwable, AddonMeta] = + JsonParser.decode[AddonMeta](str) + + def fromJsonBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromJsonString) + .rethrow + + def fromYamlString(str: String): Either[Throwable, AddonMeta] = + YamlParser.parse(str).flatMap(_.as[AddonMeta]) + + def fromYamlBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromYamlString) + .rethrow + + def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = { + val logger = docspell.logging.getLogger[F] + val jsonFile = dir / "docspell-addon.json" + val yamlFile = dir / "docspell-addon.yaml" + val yamlFile2 = dir / "docspell-addon.yml" + + OptionT + .liftF(Files[F].exists(jsonFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading json addon file $jsonFile"))) + .flatMap(OptionT.whenF(_)(fromJsonBytes(Files[F].readAll(jsonFile)))) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile)))) + ) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile2)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile2"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile2)))) + ) + .getOrElseF( + Sync[F].raiseError( + new FileNotFoundException(s"No docspell-addon.{yaml|json} file found in $dir!") + ) + ) + } + + def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = { + val fail: F[AddonMeta] = Async[F].raiseError( + new FileNotFoundException( + s"No docspell-addon.{yaml|json} file found in zip!" + ) + ) + zipFile + .through(Zip.unzip(8192, Glob("**/docspell-addon.*"))) + .filter(bin => !bin.name.endsWith("/")) + .flatMap { bin => + if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data)) + else if (bin.extensionIn(Set("yaml", "yml"))) + Stream.eval(AddonMeta.fromYamlBytes(bin.data)) + else Stream.empty + } + .take(1) + .compile + .last + .flatMap(_.map(Sync[F].pure).getOrElse(fail)) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala new file mode 100644 index 00000000..37bc0106 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala @@ -0,0 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +case class AddonRef(archive: AddonArchive, args: String) diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala new file mode 100644 index 00000000..7efa9c26 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid + +import docspell.addons.out.AddonOutput + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Codec, Decoder, Encoder} + +sealed trait AddonResult { + def toEither: Either[Throwable, AddonOutput] + + def isSuccess: Boolean = toEither.isRight + def isFailure: Boolean = !isSuccess + + def cast: AddonResult = this +} + +object AddonResult { + + /** The addon was run successful, but decoding its stdout failed. */ + case class DecodingError(message: String) extends AddonResult { + def toEither = Left(new IllegalStateException(message)) + } + object DecodingError { + implicit val jsonEncoder: Encoder[DecodingError] = deriveEncoder + implicit val jsonDecoder: Decoder[DecodingError] = deriveDecoder + } + + def decodingError(message: String): AddonResult = + DecodingError(message) + + def decodingError(ex: Throwable): AddonResult = + DecodingError(ex.getMessage) + + /** Running the addon resulted in an invalid return code (!= 0). */ + case class ExecutionError(rc: Int) extends AddonResult { + def toEither = Left(new IllegalStateException(s"Exit code: $rc")) + } + + object ExecutionError { + implicit val jsonEncoder: Encoder[ExecutionError] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionError] = deriveDecoder + } + + def executionError(rc: Int): AddonResult = + ExecutionError(rc) + + /** The execution of the addon failed with an exception. */ + case class ExecutionFailed(error: Throwable) extends AddonResult { + def toEither = Left(error) + } + + object ExecutionFailed { + implicit val throwableCodec: Codec[Throwable] = + Codec.from( + Decoder[String].emap(str => Right(ErrorMessageThrowable(str))), + Encoder[String].contramap(_.getMessage) + ) + + implicit val jsonEncoder: Encoder[ExecutionFailed] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionFailed] = deriveDecoder + + private class ErrorMessageThrowable(msg: String) extends RuntimeException(msg) { + override def fillInStackTrace() = this + } + private object ErrorMessageThrowable { + def apply(str: String): Throwable = new ErrorMessageThrowable(str) + } + } + + def executionFailed(error: Throwable): AddonResult = + ExecutionFailed(error) + + /** The addon was run successfully and its output was decoded (if any). */ + case class Success(output: AddonOutput) extends AddonResult { + def toEither = Right(output) + } + + object Success { + implicit val jsonEncoder: Encoder[Success] = deriveEncoder + implicit val jsonDecoder: Decoder[Success] = deriveDecoder + } + + def success(output: AddonOutput): AddonResult = + Success(output) + + val empty: AddonResult = Success(AddonOutput.empty) + + def combine(a: AddonResult, b: AddonResult): AddonResult = + (a, b) match { + case (Success(o1), Success(o2)) => Success(AddonOutput.combine(o1, o2)) + case (Success(_), e) => e + case (e, Success(_)) => e + case _ => a + } + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("result").withKebabCaseConstructorNames + + implicit val jsonDecoder: Decoder[AddonResult] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonResult] = deriveConfiguredEncoder + + implicit val addonResultMonoid: Monoid[AddonResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala new file mode 100644 index 00000000..75efe4a2 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Applicative +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.runner._ +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonRunner[F[_]] { + def runnerType: List[RunnerType] + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] +} + +object AddonRunner { + def forType[F[_]: Async](cfg: AddonExecutorConfig)(rt: RunnerType) = + rt match { + case RunnerType.NixFlake => NixFlakeRunner[F](cfg) + case RunnerType.Docker => DockerRunner[F](cfg) + case RunnerType.Trivial => TrivialRunner[F](cfg) + } + + def failWith[F[_]](errorMsg: String)(implicit F: Applicative[F]): AddonRunner[F] = + pure(AddonResult.executionFailed(new Exception(errorMsg))) + + def pure[F[_]: Applicative](result: AddonResult): AddonRunner[F] = + new AddonRunner[F] { + val runnerType = Nil + + def run(logger: Logger[F], env: Env, ctx: Context) = + Applicative[F].pure(result) + } + + def firstSuccessful[F[_]: Sync](runners: List[AddonRunner[F]]): AddonRunner[F] = + runners match { + case Nil => failWith("No runner available!") + case a :: Nil => a + case _ => + new AddonRunner[F] { + val runnerType = runners.flatMap(_.runnerType).distinct + + def run(logger: Logger[F], env: Env, ctx: Context) = + Stream + .emits(runners) + .evalTap(r => + logger.info( + s"Attempt to run addon ${ctx.meta.nameAndVersion} with runner ${r.runnerType}" + ) + ) + .evalMap(_.run(logger, env, ctx)) + .flatMap { + case r @ AddonResult.Success(_) => Stream.emit(r.cast.some) + case r @ AddonResult.ExecutionFailed(ex) => + if (ctx.meta.isPure) { + logger.stream + .warn(ex)(s"Addon runner failed, try next.") + .as(r.cast.some) + } else { + logger.stream.warn(ex)(s"Addon runner failed!").as(None) + } + case r @ AddonResult.ExecutionError(rc) => + if (ctx.meta.isPure) { + logger.stream + .warn(s"Addon runner returned non-zero: $rc. Try next.") + .as(r.cast.some) + } else { + logger.stream.warn(s"Addon runner returned non-zero: $rc!").as(None) + } + case AddonResult.DecodingError(message) => + // Don't retry as it is very unlikely that the output differs using another runner + // This is most likely a bug in the addon + logger.stream + .warn( + s"Error decoding the output of the addon ${ctx.meta.nameAndVersion}: $message. Stopping here. This is likely a bug in the addon." + ) + .as(None) + } + .unNoneTerminate + .takeThrough(_.isFailure) + .compile + .last + .flatMap { + case Some(r) => r.pure[F] + case None => + AddonResult + .executionFailed(new NoSuchElementException("No runner left :(")) + .pure[F] + } + } + } + + def firstSuccessful[F[_]: Sync]( + runner: AddonRunner[F], + runners: AddonRunner[F]* + ): AddonRunner[F] = + firstSuccessful(runner :: runners.toList) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala new file mode 100644 index 00000000..85438f09 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList + +import io.circe.{Decoder, Encoder} + +sealed trait AddonTriggerType { + def name: String +} + +object AddonTriggerType { + + /** The final step when processing an item. */ + case object FinalProcessItem extends AddonTriggerType { + val name = "final-process-item" + } + + /** The final step when reprocessing an item. */ + case object FinalReprocessItem extends AddonTriggerType { + val name = "final-reprocess-item" + } + + /** Running periodically based on a schedule. */ + case object Scheduled extends AddonTriggerType { + val name = "scheduled" + } + + /** Running (manually) on some existing item. */ + case object ExistingItem extends AddonTriggerType { + val name = "existing-item" + } + + val all: NonEmptyList[AddonTriggerType] = + NonEmptyList.of(FinalProcessItem, FinalReprocessItem, Scheduled, ExistingItem) + + def fromString(str: String): Either[String, AddonTriggerType] = + all + .find(e => e.name.equalsIgnoreCase(str)) + .toRight(s"Invalid addon trigger type: $str") + + def unsafeFromString(str: String): AddonTriggerType = + fromString(str).fold(sys.error, identity) + + implicit val jsonEncoder: Encoder[AddonTriggerType] = + Encoder.encodeString.contramap(_.name) + + implicit val jsonDecoder: Decoder[AddonTriggerType] = + Decoder.decodeString.emap(fromString) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Context.scala b/modules/addonlib/src/main/scala/docspell/addons/Context.scala new file mode 100644 index 00000000..1c215e24 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Context.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import fs2.io.file.Path + +import docspell.common._ +import docspell.common.exec.{Args, Env, SysCmd} + +/** Context a list of addons is executed in. + * + * Each addon has its own `addonDir`, but all share the same `baseDir` in one run. + */ +case class Context( + addon: AddonRef, + meta: AddonMeta, + baseDir: Path, + addonDir: Path, + outputDir: Path, + cacheDir: Path +) { + def userInputFile = Context.userInputFile(baseDir) + def tempDir = Context.tempDir(baseDir) + + private[addons] def addonCommand( + binary: String, + timeout: Duration, + relativeToBase: Boolean, + outputDir: Option[String], + cacheDir: Option[String] + ): SysCmd = { + val execBin = Option + .when(relativeToBase)(binary) + .getOrElse((baseDir / binary).toString) + + val input = Option + .when(relativeToBase)(baseDir.relativize(userInputFile)) + .getOrElse(userInputFile) + + val allArgs = + Args(meta.args.getOrElse(Nil)).append(input) + val envAddonDir = Option + .when(relativeToBase)(baseDir.relativize(addonDir)) + .getOrElse(addonDir) + val envTmpDir = Option + .when(relativeToBase)(baseDir.relativize(tempDir)) + .getOrElse(tempDir) + val outDir = outputDir.getOrElse(this.outputDir.toString) + val cache = cacheDir.getOrElse(this.cacheDir.toString) + val moreEnv = + Env.of( + "ADDON_DIR" -> envAddonDir.toString, + "TMPDIR" -> envTmpDir.toString, + "TMP_DIR" -> envTmpDir.toString, + "OUTPUT_DIR" -> outDir, + "CACHE_DIR" -> cache + ) + + SysCmd(execBin, allArgs).withTimeout(timeout).addEnv(moreEnv) + } +} + +object Context { + def userInputFile(base: Path): Path = + base / "arguments" / "user-input" + def tempDir(base: Path): Path = + base / "temp" +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Directory.scala b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala new file mode 100644 index 00000000..3b72ac4c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import cats.{Applicative, Monad} +import fs2.io.file.{Files, Path, PosixPermissions} + +object Directory { + + def create[F[_]: Files: Applicative](dir: Path): F[Path] = + Files[F] + .createDirectories(dir, PosixPermissions.fromOctal("777")) + .as(dir) + + def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] = + (dir :: dirs.toList).traverse_(Files[F].createDirectories(_)) + + def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + List( + Files[F].isDirectory(dir), + Files[F].list(dir).take(1).compile.last.map(_.isDefined) + ).sequence.map(_.forall(identity)) + + def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + nonEmpty(dir).map(b => !b) + + def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + d <- mkTemp(parent, prefix) + } yield d + + def temp2[F[_]: Files]( + parent: Path, + prefix1: String, + prefix2: String + ): Resource[F, (Path, Path)] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + a <- mkTemp(parent, prefix1) + b <- mkTemp(parent, prefix2) + } yield (a, b) + + def createTemp[F[_]: Files: Monad]( + parent: Path, + prefix: String + ): F[Path] = + for { + _ <- Files[F].createDirectories(parent) + d <- mkTemp_(parent, prefix) + } yield d + + private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + Files[F] + .tempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) + + private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] = + Files[F] + .createTempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala new file mode 100644 index 00000000..bd5fb7cc --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect.Resource +import fs2.io.file.{Files, Path} + +import docspell.common.exec.Env + +case class InputEnv( + addons: List[AddonRef], + baseDir: Path, + outputDir: Path, + cacheDir: Path, + env: Env +) { + def addEnv(key: String, value: String): InputEnv = + copy(env = env.add(key, value)) + + def addEnv(vp: (String, String)*): InputEnv = + copy(env = env.addAll(vp.toMap)) + + def addEnv(vm: Map[String, String]): InputEnv = + copy(env = env ++ Env(vm)) + + def withTempBase[F[_]: Files]: Resource[F, InputEnv] = + Directory.temp(baseDir, "addon-").map(path => copy(baseDir = path)) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala new file mode 100644 index 00000000..4ca1a9fb --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monad +import cats.data.Kleisli +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.Files + +trait Middleware[F[_]] extends (AddonExec[F] => AddonExec[F]) { self => + + def >>(next: Middleware[F]): Middleware[F] = + Middleware(self.andThen(next)) +} + +object Middleware { + def apply[F[_]](f: AddonExec[F] => AddonExec[F]): Middleware[F] = + a => f(a) + + def identity[F[_]]: Middleware[F] = Middleware(scala.Predef.identity) + + /** Uses a temporary base dir that is removed after execution. Use this as the last + * layer! + */ + def ephemeralRun[F[_]: Files: Sync]: Middleware[F] = + Middleware(a => Kleisli(_.withTempBase.use(a.run))) + + /** Prepare running an addon */ + def prepare[F[_]: Monad]( + prep: Kleisli[F, InputEnv, InputEnv] + ): Middleware[F] = + Middleware(a => Kleisli(in => prep.run(in).flatMap(a.run))) + + def postProcess[F[_]: Monad]( + post: Kleisli[F, AddonExecutionResult, Unit] + ): Middleware[F] = + Middleware(_.flatMapF(r => post.map(_ => r).run(r))) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala new file mode 100644 index 00000000..adf343e1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList +import cats.syntax.all._ + +import io.circe.{Decoder, Encoder} + +sealed trait RunnerType { + def name: String + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A +} +object RunnerType { + case object NixFlake extends RunnerType { + val name = "nix-flake" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = nixFlake + } + case object Docker extends RunnerType { + val name = "docker" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = docker + } + case object Trivial extends RunnerType { + val name = "trivial" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = trivial + } + + val all: NonEmptyList[RunnerType] = + NonEmptyList.of(NixFlake, Docker, Trivial) + + def fromString(str: String): Either[String, RunnerType] = + all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid runner value: $str") + + def unsafeFromString(str: String): RunnerType = + fromString(str).fold(sys.error, identity) + + def fromSeparatedString(str: String): Either[String, List[RunnerType]] = + str.split("[\\s,]+").toList.map(_.trim).traverse(fromString) + + implicit val jsonDecoder: Decoder[RunnerType] = + Decoder[String].emap(RunnerType.fromString) + + implicit val jsonEncoder: Encoder[RunnerType] = + Encoder[String].contramap(_.name) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala new file mode 100644 index 00000000..143e3dac --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.kernel.Monoid + +import docspell.common.bc.BackendCommand + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Decoded stdout result from executing an addon. */ +case class AddonOutput( + commands: List[BackendCommand] = Nil, + files: List[ItemFile] = Nil, + newItems: List[NewItem] = Nil +) + +object AddonOutput { + val empty: AddonOutput = AddonOutput() + + def combine(a: AddonOutput, b: AddonOutput): AddonOutput = + AddonOutput(a.commands ++ b.commands, a.files ++ b.files) + + implicit val addonResultMonoid: Monoid[AddonOutput] = + Monoid.instance(empty, combine) + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonDecoder: Decoder[AddonOutput] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonOutput] = deriveConfiguredEncoder + + def fromString(str: String): Either[Throwable, AddonOutput] = + io.circe.parser.decode[AddonOutput](str) + + def unsafeFromString(str: String): AddonOutput = + fromString(str).fold(throw _, identity) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala new file mode 100644 index 00000000..bd470613 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.FileSupport._ +import docspell.logging.Logger + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Addons can produce files in their output directory. These can be named here in order + * to do something with them. + * + * - textFiles will replace the extracted text with the contents of the file + * - pdfFiles will add/replace the converted pdf with the given file + * - previewImages will add/replace preview images + * - newFiles will be added as new attachments to the item + * + * Files must be referenced by attachment id. + */ +final case class ItemFile( + itemId: Ident, + textFiles: Map[String, String] = Map.empty, + pdfFiles: Map[String, String] = Map.empty, + previewImages: Map[String, String] = Map.empty, + newFiles: List[NewFile] = Nil +) { + def isEmpty: Boolean = + textFiles.isEmpty && pdfFiles.isEmpty && previewImages.isEmpty + + def nonEmpty: Boolean = !isEmpty + + def resolveTextFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.text("*"), textFiles) + + def resolvePdfFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.pdf, pdfFiles) + + def resolvePreviewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.image("*"), previewImages) + + def resolveNewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(NewFile, Path)]] = + newFiles.traverseFilter(nf => + nf.resolveFile(logger, outputDir).map(_.map(p => (nf, p))) + ) + + private def resolveFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path, + mime: MimeType, + files: Map[String, String] + ): F[List[(String, Path)]] = { + val allFiles = + files.toList.map(t => t._1 -> outputDir / t._2) + + allFiles.traverseFilter { case (key, file) => + OptionT(file.detectMime) + .flatMapF(fileType => + if (mime.matches(fileType)) (key -> file).some.pure[F] + else + logger + .warn( + s"File $file provided as ${mime.asString} file, but was recognized as ${fileType.asString}. Ignoring it." + ) + .as(None: Option[(String, Path)]) + ) + .value + } + } +} + +object ItemFile { + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonEncoder: Encoder[ItemFile] = deriveConfiguredEncoder + implicit val jsonDecoder: Decoder[ItemFile] = deriveConfiguredDecoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala new file mode 100644 index 00000000..4807fe9c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.effect.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewFile.Meta +import docspell.common.ProcessItemArgs.ProcessMeta +import docspell.common.{Ident, Language} +import docspell.logging.Logger + +import io.circe.Codec +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.deriveConfiguredCodec +import io.circe.generic.semiauto.deriveCodec + +case class NewFile(metadata: Meta = Meta.empty, file: String) { + + def resolveFile[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[Option[Path]] = { + val target = outputDir / file + Files[F] + .exists(target) + .flatMap(flag => + if (flag) target.some.pure[F] + else logger.warn(s"File not found: $file").as(Option.empty) + ) + } +} + +object NewFile { + + case class Meta( + language: Option[Language], + skipDuplicate: Option[Boolean], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessMeta( + cid: Ident, + itemId: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessMeta = + ProcessMeta( + collective = cid, + itemId = Some(itemId), + language = language.orElse(collLang).getOrElse(Language.English), + direction = None, + sourceAbbrev = sourceAbbrev, + folderId = None, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = None, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + val empty = Meta(None, None, None) + implicit val jsonCodec: Codec[Meta] = deriveCodec + } + + implicit val jsonConfig: Configuration = Configuration.default.withDefaults + + implicit val jsonCodec: Codec[NewFile] = deriveConfiguredCodec +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala new file mode 100644 index 00000000..c5511fd0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.Monad +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewItem.Meta +import docspell.common._ +import docspell.logging.Logger + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +case class NewItem(metadata: Option[Meta], files: List[String]) { + + def toProcessMeta( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + metadata + .getOrElse(Meta(None, None, None, None, None, None, None)) + .toProcessArgs(cid, collLang, sourceAbbrev) + + def resolveFiles[F[_]: Files: Monad]( + logger: Logger[F], + outputDir: Path + ): F[List[Path]] = { + val allFiles = + files.map(name => outputDir / name) + + allFiles.traverseFilter { file => + Files[F] + .exists(file) + .flatMap { + case true => file.some.pure[F] + case false => + logger + .warn(s"File $file doesn't exist. Ignoring it.") + .as(None) + } + } + } +} + +object NewItem { + + case class Meta( + language: Option[Language], + direction: Option[Direction], + folderId: Option[Ident], + source: Option[String], + skipDuplicate: Option[Boolean], + tags: Option[List[String]], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessArgs( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + ProcessItemArgs.ProcessMeta( + collective = cid, + itemId = None, + language = language.orElse(collLang).getOrElse(Language.English), + direction = direction, + sourceAbbrev = source.getOrElse(sourceAbbrev), + folderId = folderId, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = tags, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = deriveDecoder + } + + implicit val jsonDecoder: Decoder[NewItem] = deriveDecoder + implicit val jsonEncoder: Encoder[NewItem] = deriveEncoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/package.scala b/modules/addonlib/src/main/scala/docspell/addons/package.scala new file mode 100644 index 00000000..82a67232 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/package.scala @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell + +import cats.data.Kleisli + +package object addons { + + type AddonExec[F[_]] = Kleisli[F, InputEnv, AddonExecutionResult] + +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala new file mode 100644 index 00000000..48fa5b75 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.Applicative +import cats.effect.{Ref, Sync} +import cats.syntax.all._ +import fs2.Pipe + +trait CollectOut[F[_]] { + + def get: F[String] + + def append: Pipe[F, String, String] +} + +object CollectOut { + + def none[F[_]: Applicative]: CollectOut[F] = + new CollectOut[F] { + def get = "".pure[F] + def append = identity + } + + def buffer[F[_]: Sync]: F[CollectOut[F]] = + Ref + .of[F, Vector[String]](Vector.empty) + .map(buffer => + new CollectOut[F] { + override def get = + buffer.get.map(_.mkString("\n").trim) + + override def append = + _.evalTap(line => + if (line.trim.nonEmpty) buffer.update(_.appended(line)) else ().pure[F] + ) + } + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala new file mode 100644 index 00000000..b41d46e0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} + +/** Builder for a docker system command. */ +case class DockerBuilder( + dockerBinary: String, + subCmd: String, + timeout: Duration, + containerName: Option[String] = None, + env: Env = Env.empty, + mounts: Args = Args.empty, + network: Option[String] = Some("host"), + workingDir: Option[String] = None, + imageName: Option[String] = None, + cntCmd: Args = Args.empty +) { + def containerCmd(args: Args): DockerBuilder = + copy(cntCmd = args) + def containerCmd(args: Seq[String]): DockerBuilder = + copy(cntCmd = Args(args)) + + def imageName(name: String): DockerBuilder = + copy(imageName = Some(name)) + + def workDirectory(dir: String): DockerBuilder = + copy(workingDir = Some(dir)) + + def withDockerBinary(bin: String): DockerBuilder = + copy(dockerBinary = bin) + + def withSubCmd(cmd: String): DockerBuilder = + copy(subCmd = cmd) + + def withEnv(key: String, value: String): DockerBuilder = + copy(env = env.add(key, value)) + + def withEnv(moreEnv: Env): DockerBuilder = + copy(env = env ++ moreEnv) + + def privateNetwork(flag: Boolean): DockerBuilder = + if (flag) copy(network = Some("none")) + else copy(network = Some("host")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): DockerBuilder = { + val target = cntDir.getOrElse(hostDir.toString) + val ro = Option.when(readOnly)(",readonly").getOrElse("") + val opt = s"type=bind,source=$hostDir,target=$target${ro}" + copy(mounts = mounts.append("--mount", opt)) + } + + def withName(containerName: String): DockerBuilder = + copy(containerName = Some(containerName)) + + def build: SysCmd = + SysCmd(dockerBinary, buildArgs).withTimeout(timeout) + + private def buildArgs: Args = + Args + .of(subCmd) + .append("--rm") + .option("--name", containerName) + .append(mounts) + .option("--network", network) + .append(env.mapConcat((k, v) => List("--env", s"${k}=${v}"))) + .option("-w", workingDir) + .appendOpt(imageName) + .append(cntCmd) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala new file mode 100644 index 00000000..a20ab5a0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonExecutorConfig.DockerConfig +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Env, SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +final class DockerRunner[F[_]: Async](cfg: DockerRunner.Config) extends AddonRunner[F] { + + val runnerType = List(RunnerType.Docker) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = for { + _ <- OptionT.whenF(requireBuild(ctx))(build(logger, ctx)).value + suffix <- Random[F].string(4) + cmd = createDockerCommand(env, ctx, suffix) + result <- RunnerUtil.runAddonCommand(logger, cmd, ctx) + } yield result + + def createDockerCommand( + env: Env, + ctx: Context, + suffix: String + ): SysCmd = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonArgs = + ctx.addonCommand( + "", + Duration.zero, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + + DockerBuilder(cfg.docker.dockerBinary, "run", cfg.timeout) + .withName(ctx.meta.nameAndVersion + "-" + suffix) + .withEnv(env) + .withEnv(addonArgs.env) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .workDirectory("/mnt/work") + .privateNetwork(ctx.meta.isPure) + .imageName(imageName(ctx)) + .containerCmd(addonArgs.args) + .build + } + + def build(logger: Logger[F], ctx: Context): F[Unit] = + for { + _ <- logger.info(s"Building docker image for addon ${ctx.meta.nameAndVersion}") + cmd = cfg.docker.dockerBuild(imageName(ctx)) + _ <- SysExec(cmd, logger, ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "docker build")) + .use(_.waitFor()) + _ <- logger.info(s"Docker image built successfully") + } yield () + + private def requireBuild(ctx: Context) = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .isEmpty + + private def imageName(ctx: Context): String = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .getOrElse(s"${ctx.meta.meta.name}:latest") +} + +object DockerRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): DockerRunner[F] = + new DockerRunner[F](Config(cfg.dockerRunner, cfg.runTimeout)) + + case class Config(docker: DockerConfig, timeout: Duration) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala new file mode 100644 index 00000000..accd2059 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.exec.{Args, Env, SysCmd} + +case class NSpawnBuilder( + child: SysCmd, + chroot: Path, + spawnBinary: String = "systemd-nspawn", + sudoBinary: String = "sudo", + args: Args = Args.empty, + env: Env = Env.empty +) { + + def withNSpawnBinary(bin: String): NSpawnBuilder = + copy(spawnBinary = bin) + + def withSudoBinary(bin: String): NSpawnBuilder = + copy(sudoBinary = bin) + + def withEnv(key: String, value: String): NSpawnBuilder = + copy(args = args.append(s"--setenv=$key=$value")) + + def withEnvOpt(key: String, value: Option[String]): NSpawnBuilder = + value.map(v => withEnv(key, v)).getOrElse(this) + + def withName(containerName: String): NSpawnBuilder = + copy(args = args.append(s"--machine=$containerName")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): NSpawnBuilder = { + val bind = if (readOnly) "--bind-ro" else "--bind" + val target = cntDir.map(dir => s":$dir").getOrElse("") + copy(args = args.append(s"${bind}=${hostDir}${target}")) + } + + def workDirectory(dir: String): NSpawnBuilder = + copy(args = args.append(s"--chdir=$dir")) + + def portMap(port: Int): NSpawnBuilder = + copy(args = args.append("-p", port.toString)) + + def privateNetwork(flag: Boolean): NSpawnBuilder = + if (flag) copy(args = args.append("--private-network")) + else this + + def build: SysCmd = + SysCmd( + program = if (sudoBinary.nonEmpty) sudoBinary else spawnBinary, + args = buildArgs, + timeout = child.timeout, + env = env + ) + + private def buildArgs: Args = + Args + .of("--private-users=identity") // can't use -U because need writeable bind mounts + .append("--notify-ready=yes") + .append("--ephemeral") + .append("--as-pid2") + .append("--console=pipe") + .append("--no-pager") + .append("--bind-ro=/bin") + .append("--bind-ro=/usr/bin") + .append("--bind-ro=/nix/store") + .append(s"--directory=$chroot") + .append(args) + .append(child.env.map((n, v) => s"--setenv=$n=$v")) + .prependWhen(sudoBinary.nonEmpty)(spawnBinary) + .append("--") + .append(child.program) + .append(child.args) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala new file mode 100644 index 00000000..ce12c73c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala @@ -0,0 +1,123 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.addons.AddonExecutorConfig.{NSpawn, NixConfig} +import docspell.addons._ +import docspell.addons.runner.NixFlakeRunner.PreCtx +import docspell.common.Duration +import docspell.common.exec._ +import docspell.logging.Logger + +final class NixFlakeRunner[F[_]: Async](cfg: NixFlakeRunner.Config) + extends AddonRunner[F] { + + val runnerType = List(RunnerType.NixFlake) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] = + prepare(logger, ctx) + .flatMap { preCtx => + if (preCtx.nspawnEnabled) runInContainer(logger, env, preCtx, ctx) + else runOnHost(logger, env, preCtx, ctx) + } + + def prepare(logger: Logger[F], ctx: Context): F[PreCtx] = + for { + _ <- logger.info(s"Prepare addon ${ctx.meta.nameAndVersion} for executing via nix") + _ <- logger.debug(s"Building with nix build") + _ <- SysExec(cfg.nixBuild, logger, workdir = ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "nix build")) + .use(_.waitFor()) + bin <- findFile(ctx.addonDir / "result" / "bin", ctx.addonDir / "result") + _ <- logger.debug(s"Build done, found binary: $bin") + _ <- logger.debug(s"Checking for systemd-nspawn…") + cnt <- checkContainer(logger) + _ <- + if (cnt) + logger.debug(s"Using systemd-nspawn to run addon in a container.") + else + logger.info(s"Running via systemd-nspawn is disabled in the config file") + } yield PreCtx(cnt, ctx.baseDir.relativize(bin)) + + private def checkContainer(logger: Logger[F]): F[Boolean] = + if (!cfg.nspawn.enabled) false.pure[F] + else RunnerUtil.checkContainer(logger, cfg.nspawn) + + private def runOnHost( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = + SysCmd(preCtx.binary.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + + private def runInContainer( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = SysCmd(preCtx.binary.toString, Args.empty) + .withTimeout(cfg.timeout) + .addEnv(env) + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + } + + /** Find first file, try directories in given order. */ + private def findFile(firstDir: Path, more: Path*): F[Path] = { + val fail: F[Path] = Sync[F].raiseError( + new NoSuchElementException( + s"No file found to execute in ${firstDir :: more.toList}" + ) + ) + + Stream + .emits(more) + .cons1(firstDir) + .flatMap(dir => + Files[F] + .list(dir) + .evalFilter(p => Files[F].isDirectory(p).map(!_)) + .take(1) + ) + .take(1) + .compile + .last + .flatMap(_.fold(fail)(Sync[F].pure)) + } +} + +object NixFlakeRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): NixFlakeRunner[F] = + new NixFlakeRunner[F](Config(cfg.nixRunner, cfg.nspawn, cfg.runTimeout)) + + case class Config( + nix: NixConfig, + nspawn: NSpawn, + timeout: Duration + ) { + + val nixBuild = + SysCmd(nix.nixBinary, Args.of("build")).withTimeout(nix.buildTimeout) + + val nspawnVersion = nspawn.nspawnVersion + } + + case class PreCtx(nspawnEnabled: Boolean, binary: Path) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala new file mode 100644 index 00000000..e404b442 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala @@ -0,0 +1,171 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect.{Async, Sync} +import cats.syntax.all._ +import fs2.Pipe +import fs2.io.file.Files + +import docspell.addons._ +import docspell.addons.out.AddonOutput +import docspell.common.exec.{SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +import io.circe.{parser => JsonParser} + +private[addons] object RunnerUtil { + + /** Run the given `cmd` on this machine. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runDirectly[F[_]: Async]( + logger: Logger[F], + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val addonCmd = ctx + .addonCommand(cmd.program, cmd.timeout, relativeToBase = false, None, None) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + runAddonCommand(logger, addonCmd, ctx) + } + + /** Run the given `cmd` inside a container via systemd-nspawn. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runInContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn, + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonCmd = ctx + .addonCommand( + cmd.program, + cmd.timeout, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + + val chroot = ctx.baseDir / "cnt-root" + val nspawn = NSpawnBuilder(addonCmd, chroot) + .withNSpawnBinary(cfg.nspawnBinary) + .withSudoBinary(cfg.sudoBinary) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .workDirectory("/mnt/work") + .withEnv("XDG_RUNTIME_DIR", "/mnt/work") + .privateNetwork(ctx.meta.isPure) + + for { + suffix <- Random[F].string(4) + _ <- List(chroot).traverse_(Files[F].createDirectories) + res <- runAddonCommand( + logger, + nspawn.withName(ctx.meta.nameAndVersion + "-" + suffix).build, + ctx + ) + // allow some time to unregister the current container + // only important when same addons are called in sequence too fast + _ <- Sync[F].sleep(cfg.containerWait.toScala) + } yield res + } + + private def procPipe[F[_]]( + p: String, + ctx: Context, + collect: CollectOut[F], + logger: Logger[F] + ): Pipe[F, String, Unit] = + _.through(collect.append) + .map(line => s">> [${ctx.meta.nameAndVersion} ($p)] $line") + .evalMap(logger.debug(_)) + + /** Runs the external command that is executing the addon. + * + * If the addons specifies to collect its output, the stdout is parsed as json and + * decoded into [[AddonOutput]]. + */ + def runAddonCommand[F[_]: Async]( + logger: Logger[F], + cmd: SysCmd, + ctx: Context + ): F[AddonResult] = + for { + stdout <- + if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F] + else CollectOut.none[F].pure[F] + cmdResult <- SysExec(cmd, logger, ctx.baseDir.some) + .flatMap( + _.consumeOutputs( + procPipe("out", ctx, stdout, logger), + procPipe("err", ctx, CollectOut.none[F], logger) + ) + ) + .use(_.waitFor()) + .attempt + addonResult <- cmdResult match { + case Right(rc) if rc != 0 => + for { + _ <- logger.error( + s"Addon ${ctx.meta.nameAndVersion} returned non-zero: $rc" + ) + } yield AddonResult.executionError(rc) + + case Right(_) => + for { + _ <- logger.debug(s"Addon ${ctx.meta.nameAndVersion} executed successfully!") + out <- stdout.get + _ <- logger.debug(s"Addon stdout: $out") + result = Option + .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)( + JsonParser + .decode[AddonOutput](out) + .fold(AddonResult.decodingError, AddonResult.success) + ) + .getOrElse(AddonResult.empty) + } yield result + + case Left(ex) => + logger + .error(ex)(s"Executing external command failed!") + .as(AddonResult.executionFailed(ex)) + } + } yield addonResult + + /** Check whether `systemd-nspawn` is available on this machine. */ + def checkContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn + ): F[Boolean] = + for { + rc <- SysExec(cfg.nspawnVersion, logger) + .flatMap(_.logOutputs(logger, "nspawn")) + .use(_.waitFor()) + _ <- + OptionT + .whenF(rc != 0)( + logger.warn( + s"No systemd-nspawn found! Addon is not executed inside a container." + ) + ) + .value + } yield rc == 0 +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala new file mode 100644 index 00000000..31ac2694 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.kernel.Monoid +import cats.syntax.all._ +import fs2.io.file.PosixPermission._ +import fs2.io.file.{Files, PosixPermissions} + +import docspell.addons.AddonExecutorConfig.NSpawn +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} +import docspell.logging.Logger + +final class TrivialRunner[F[_]: Async](cfg: TrivialRunner.Config) extends AddonRunner[F] { + private val sync = Async[F] + private val files = Files[F] + implicit val andMonoid: Monoid[Boolean] = Monoid.instance[Boolean](true, _ && _) + + private val executeBits = PosixPermissions( + OwnerExecute, + OwnerRead, + OwnerWrite, + GroupExecute, + GroupRead, + OthersExecute, + OthersRead + ) + + val runnerType = List(RunnerType.Trivial) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = { + val binaryPath = ctx.meta.runner + .flatMap(_.trivial) + .map(_.exec) + .map(bin => ctx.addonDir / bin) + + binaryPath match { + case None => + sync.raiseError(new IllegalStateException("No executable specified in addon!")) + + case Some(file) => + val bin = ctx.baseDir.relativize(file) + val cmd = SysCmd(bin.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + + val withNSpawn = + OptionT + .whenF(cfg.nspawn.enabled)(RunnerUtil.checkContainer(logger, cfg.nspawn)) + .getOrElse(false) + + files.setPosixPermissions(file, executeBits).attempt *> + withNSpawn.flatMap { + case true => + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + case false => + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + } + } +} + +object TrivialRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): TrivialRunner[F] = + new TrivialRunner[F](Config(cfg.nspawn, cfg.runTimeout)) + + case class Config(nspawn: NSpawn, timeout: Duration) +} diff --git a/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip b/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip new file mode 100644 index 00000000..20482861 Binary files /dev/null and b/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip differ diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala new file mode 100644 index 00000000..702292d2 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.option._ + +import docspell.common.UrlReader +import docspell.logging.TestLoggingConfig + +import munit._ + +class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtures { + val logger = docspell.logging.getLogger[IO] + + tempDir.test("Read archive from directory") { dir => + for { + archive <- IO(AddonArchive(dummyAddonUrl, "", "")) + path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir) + + aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some) + _ = { + assertEquals(aa.name, "dummy-addon") + assertEquals(aa.version, "2.9") + assertEquals(aa.url, dummyAddonUrl) + } + } yield () + } + + test("Read archive from zip file") { + for { + archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO]) + _ = { + assertEquals(archive.name, "dummy-addon") + assertEquals(archive.version, "2.9") + assertEquals(archive.url, dummyAddonUrl) + } + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala new file mode 100644 index 00000000..5ad59b14 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.logging.{Level, TestLoggingConfig} + +import munit._ + +class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + + override def docspellLogConfig = + super.docspellLogConfig.copy(minimumLevel = Level.Trace) + + tempDir.test("select docker if Dockerfile exists") { dir => + for { + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker)) + } yield () + } + + tempDir.test("select nix-flake if flake.nix exists") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.NixFlake)) + } yield () + } + + tempDir.test("select nix-flake and docker") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake)) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala new file mode 100644 index 00000000..9403d02c --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.common.Glob +import docspell.files.Zip +import docspell.logging.TestLoggingConfig + +import munit._ + +class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures { + val logger = docspell.logging.getLogger[IO] + + test("read meta from zip file") { + val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192)) + assertIO(meta, dummyAddonMeta) + } + + tempDir.test("read meta from directory") { dir => + for { + _ <- dummyAddonUrl + .readURL[IO](8192) + .through(Zip.unzip(8192, Glob.all)) + .through(Zip.saveTo(logger, dir, moveUp = true)) + .compile + .drain + meta <- AddonMeta.findInDirectory[IO](dir) + _ = assertEquals(meta, dummyAddonMeta) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala new file mode 100644 index 00000000..0a4bb08e --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.out.AddonOutput + +import io.circe.parser.decode +import munit.FunSuite + +class AddonOutputTest extends FunSuite { + + test("decode empty object") { + val out = decode[AddonOutput]("{}") + println(out) + } + + test("decode sample output") { + val jsonStr = + """{ "files": [ + | { + | "itemId": "qZDnyGIAJsXr", + | "textFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.txt" + | }, + | "pdfFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.pdf" + | } + | } + | ] + |} + |""".stripMargin + + val out = decode[AddonOutput](jsonStr) + println(out) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala new file mode 100644 index 00000000..1f361ae2 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala @@ -0,0 +1,98 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.util.concurrent.atomic.AtomicInteger + +import cats.effect.IO +import fs2.io.file.Path + +import docspell.common.LenientUri +import docspell.common.exec.Env +import docspell.logging.{Logger, TestLoggingConfig} + +import munit._ + +class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig { + + val logger = docspell.logging.getLogger[IO] + + val dummyContext = Context( + addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""), + meta = AddonMeta.empty("test", "1.0"), + baseDir = Path(""), + addonDir = Path(""), + outputDir = Path(""), + cacheDir = Path("") + ) + + test("firstSuccessful must stop on first success") { + val counter = new AtomicInteger(0) + val runner = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(runner, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("firstSuccessful must try with next on error") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.failWith[IO]("failed") + val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(fail, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("do not retry on decoding errors") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful(fail, increment, increment) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 0) + } yield () + } + + test("try on errors but stop on decoding error") { + val counter = new AtomicInteger(0) + val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val incrementFail = + new MockRunner(IO(counter.incrementAndGet()).void) + .as(AddonResult.executionFailed(new Exception("fail"))) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful( + incrementFail, + incrementFail, + decodeFail, + increment, + increment + ) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 2) + } yield () + } + + final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty) + extends AddonRunner[IO] { + val runnerType = Nil + def run( + logger: Logger[IO], + env: Env, + ctx: Context + ) = run.as(result) + + def as(r: AddonResult) = new MockRunner(run, r) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala new file mode 100644 index 00000000..c20ac112 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path, PosixPermissions} + +import docspell.addons.AddonExecutorConfig._ +import docspell.addons.AddonMeta._ +import docspell.addons.AddonTriggerType._ +import docspell.common.{Duration, LenientUri} +import docspell.logging.TestLoggingConfig + +import munit.CatsEffectSuite + +trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite => + + val files: Files[IO] = Files[IO] + + val dummyAddonUrl = + LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip")) + + val dummyAddonMeta = + AddonMeta( + meta = + AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some), + triggers = Some( + Set(Scheduled, FinalProcessItem, FinalReprocessItem) + ), + None, + runner = Runner( + nix = NixRunner(true).some, + docker = DockerRunner( + enable = true, + image = None, + build = "Dockerfile".some + ).some, + trivial = TrivialRunner(true, "src/addon.sh").some + ).some, + options = Options(networking = true, collectOutput = true).some + ) + + def baseTempDir: Path = + Path(s"/tmp/target/test-temp") + + val tempDir = + ResourceFixture[Path]( + Resource.eval(Files[IO].createDirectories(baseTempDir)) *> + Files[IO] + .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777")) + ) + + def testExecutorConfig( + runner: RunnerType, + runners: RunnerType* + ): AddonExecutorConfig = { + val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100)) + AddonExecutorConfig( + runner :: runners.toList, + Duration.minutes(2), + nspawn, + NixConfig("nix", Duration.minutes(2)), + DockerConfig("docker", Duration.minutes(2)) + ) + } +} diff --git a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala index 15ac3f24..9542d3bb 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala @@ -15,8 +15,8 @@ import fs2.io.file.{Files, Path} import docspell.analysis.classifier import docspell.analysis.classifier.TextClassifier._ import docspell.analysis.nlp.Properties -import docspell.common._ import docspell.common.syntax.FileSyntax._ +import docspell.common.util.File import docspell.logging.Logger import edu.stanford.nlp.classify.ColumnDataClassifier diff --git a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala index f8423492..28f9490b 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala @@ -14,6 +14,7 @@ import cats.implicits._ import docspell.analysis.NlpSettings import docspell.common._ +import docspell.common.util.File /** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and * initializing large objects. diff --git a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala index 86804836..570a8439 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala @@ -17,6 +17,7 @@ import fs2.io.file.Files import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ +import docspell.common.util.File import docspell.logging.TestLoggingConfig import munit._ diff --git a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala index d522ec6c..15bc0704 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala @@ -13,6 +13,7 @@ import cats.effect.unsafe.implicits.global import docspell.analysis.Env import docspell.common._ +import docspell.common.util.File import docspell.files.TestFiles import docspell.logging.TestLoggingConfig diff --git a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala index 015e771b..386e2a4a 100644 --- a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala +++ b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala @@ -20,6 +20,7 @@ trait AttachedEvent[R] { object AttachedEvent { + /** Only the result, no events. */ def only[R](v: R): AttachedEvent[R] = new AttachedEvent[R] { val value = v diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index a34eaf6a..cc55c72b 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -8,11 +8,14 @@ package docspell.backend import cats.effect._ +import docspell.backend.BackendCommands.EventContext import docspell.backend.auth.Login import docspell.backend.fulltext.CreateIndex import docspell.backend.ops._ import docspell.backend.signup.OSignup +import docspell.common.bc.BackendCommandRunner import docspell.ftsclient.FtsClient +import docspell.joexapi.client.JoexClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT import docspell.scheduler.JobStoreModule @@ -20,6 +23,7 @@ import docspell.store.Store import docspell.totp.Totp import emil.Emil +import org.http4s.client.Client trait BackendApp[F[_]] { @@ -35,6 +39,7 @@ trait BackendApp[F[_]] { def job: OJob[F] def item: OItem[F] def itemSearch: OItemSearch[F] + def attachment: OAttachment[F] def fulltext: OFulltext[F] def mail: OMail[F] def joex: OJoex[F] @@ -52,23 +57,30 @@ trait BackendApp[F[_]] { def fileRepository: OFileRepository[F] def itemLink: OItemLink[F] def downloadAll: ODownloadAll[F] + def addons: OAddons[F] + + def commands(eventContext: Option[EventContext]): BackendCommandRunner[F, Unit] } object BackendApp { def create[F[_]: Async]( + cfg: Config, store: Store[F], javaEmil: Emil[F], + httpClient: Client[F], ftsClient: FtsClient[F], pubSubT: PubSubT[F], schedulerModule: JobStoreModule[F], notificationMod: NotificationModule[F] ): Resource[F, BackendApp[F]] = for { + nodeImpl <- ONode(store) totpImpl <- OTotp(store, Totp.default) loginImpl <- Login[F](store, Totp.default) signupImpl <- OSignup[F](store) - joexImpl <- OJoex(pubSubT) + joexClient = JoexClient(httpClient) + joexImpl <- OJoex(pubSubT, nodeImpl, joexClient) collImpl <- OCollective[F]( store, schedulerModule.userTasks, @@ -80,7 +92,6 @@ object BackendApp { equipImpl <- OEquipment[F](store) orgImpl <- OOrganization(store) uploadImpl <- OUpload(store, schedulerModule.jobs) - nodeImpl <- ONode(store) jobImpl <- OJob(store, joexImpl, pubSubT) createIndex <- CreateIndex.resource(ftsClient, store) itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs) @@ -109,6 +120,16 @@ object BackendApp { fileRepoImpl <- OFileRepository(store, schedulerModule.jobs) itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl)) downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs)) + attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs)) + addonsImpl <- Resource.pure( + OAddons( + cfg.addons, + store, + schedulerModule.userTasks, + schedulerModule.jobs, + joexImpl + ) + ) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl @@ -139,5 +160,10 @@ object BackendApp { val fileRepository = fileRepoImpl val itemLink = itemLinkImpl val downloadAll = downloadAllImpl + val addons = addonsImpl + val attachment = attachImpl + + def commands(eventContext: Option[EventContext]) = + BackendCommands.fromBackend(this, eventContext) } } diff --git a/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala new file mode 100644 index 00000000..3550cfbf --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ + +import docspell.backend.BackendCommands.EventContext +import docspell.backend.ops.OCustomFields.SetValue +import docspell.backend.ops._ +import docspell.common.bc._ +import docspell.common.{AccountId, Ident, LenientUri} + +private[backend] class BackendCommands[F[_]: Sync]( + itemOps: OItem[F], + attachOps: OAttachment[F], + fieldOps: OCustomFields[F], + notificationOps: ONotification[F], + eventContext: Option[EventContext] +) extends BackendCommandRunner[F, Unit] { + private[this] val logger = docspell.logging.getLogger[F] + + def run(collective: Ident, cmd: BackendCommand): F[Unit] = + doRun(collective, cmd).attempt.flatMap { + case Right(_) => ().pure[F] + case Left(ex) => + logger.error(ex)(s"Backend command $cmd failed for collective ${collective.id}.") + } + + def doRun(collective: Ident, cmd: BackendCommand): F[Unit] = + cmd match { + case BackendCommand.ItemUpdate(item, actions) => + actions.traverse_(a => runItemAction(collective, item, a)) + + case BackendCommand.AttachmentUpdate(item, attach, actions) => + actions.traverse_(a => runAttachAction(collective, item, attach, a)) + } + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[Unit] = + cmds.traverse_(run(collective, _)) + + def runItemAction(collective: Ident, item: Ident, action: ItemAction): F[Unit] = + action match { + case ItemAction.AddTags(tags) => + logger.debug(s"Setting tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .linkTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.RemoveTags(tags) => + logger.debug(s"Remove tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .removeTagsMultipleItems(Nel.of(item), tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.ReplaceTags(tags) => + logger.debug(s"Replace tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .setTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.SetFolder(folder) => + logger.debug(s"Set folder $folder on ${item.id} for ${collective.id}") *> + itemOps + .setFolder(item, folder, collective) + .void + + case ItemAction.RemoveTagsCategory(cats) => + logger.debug( + s"Remove tags in categories $cats on ${item.id} for ${collective.id}" + ) *> + itemOps + .removeTagsOfCategories(item, collective, cats) + .flatMap(sendEvents) + + case ItemAction.SetCorrOrg(id) => + logger.debug( + s"Set correspondent organization ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrOrg(Nel.of(item), id, collective).void + + case ItemAction.SetCorrPerson(id) => + logger.debug( + s"Set correspondent person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcPerson(id) => + logger.debug( + s"Set concerning person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcEquipment(id) => + logger.debug( + s"Set concerning equipment ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcEquip(Nel.of(item), id, collective).void + + case ItemAction.SetField(field, value) => + logger.debug( + s"Set field on item ${item.id} ${field.id} to '$value' for ${collective.id}" + ) *> + fieldOps + .setValue(item, SetValue(field, value, collective)) + .flatMap(sendEvents) + + case ItemAction.SetNotes(notes) => + logger.debug(s"Set notes on item ${item.id} for ${collective.id}") *> + itemOps.setNotes(item, notes, collective).void + + case ItemAction.AddNotes(notes, sep) => + logger.debug(s"Add notes on item ${item.id} for ${collective.id}") *> + itemOps.addNotes(item, notes, sep, collective).void + + case ItemAction.SetName(name) => + logger.debug(s"Set name '$name' on item ${item.id} for ${collective.id}") *> + itemOps.setName(item, name, collective).void + } + + def runAttachAction( + collective: Ident, + itemId: Ident, + attachId: Ident, + action: AttachmentAction + ): F[Unit] = + action match { + case AttachmentAction.SetExtractedText(text) => + attachOps.setExtractedText( + collective, + itemId, + attachId, + text.getOrElse("").pure[F] + ) + } + + private def sendEvents(result: AttachedEvent[_]): F[Unit] = + eventContext match { + case Some(ctx) => + notificationOps.offerEvents(result.event(ctx.account, ctx.baseUrl)) + case None => ().pure[F] + } +} + +object BackendCommands { + + /** If supplied, notification events will be send. */ + case class EventContext(account: AccountId, baseUrl: Option[LenientUri]) + + def fromBackend[F[_]: Sync]( + backendApp: BackendApp[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F]( + backendApp.item, + backendApp.attachment, + backendApp.customFields, + backendApp.notification, + eventContext + ) + + def apply[F[_]: Sync]( + item: OItem[F], + attachment: OAttachment[F], + fields: OCustomFields[F], + notification: ONotification[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F](item, attachment, fields, notification, eventContext) +} diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index efccb8dc..b1baedcb 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -20,7 +20,8 @@ case class Config( mailDebug: Boolean, jdbc: JdbcConfig, signup: SignupConfig, - files: Config.Files + files: Config.Files, + addons: Config.Addons ) { def mailSettings: Settings = @@ -66,4 +67,21 @@ object Config { (storesEmpty |+| defaultStorePresent).map(_ => this) } } + + case class Addons( + enabled: Boolean, + allowImpure: Boolean, + allowedUrls: UrlMatcher, + deniedUrls: UrlMatcher + ) { + def isAllowed(url: LenientUri): Boolean = + allowedUrls.matches(url) && !deniedUrls.matches(url) + + def isDenied(url: LenientUri): Boolean = + !isAllowed(url) + } + object Addons { + val disabled: Addons = + Addons(false, false, UrlMatcher.False, UrlMatcher.True) + } } diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index 87adbbdc..cf87d783 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -16,6 +16,26 @@ import docspell.notification.api.PeriodicQueryArgs import docspell.scheduler.Job object JobFactory extends MailAddressCodec { + def existingItemAddon[F[_]: Sync]( + args: ItemAddonTaskArgs, + submitter: AccountId + ): F[Job[ItemAddonTaskArgs]] = + Job.createNew( + ItemAddonTaskArgs.taskName, + submitter.collective, + args, + "Run addons on item", + submitter.user, + Priority.High, + args.addonRunConfigs + .map(_.take(23)) + .toList + .sorted + .foldLeft(args.itemId)(_ / _) + .take(250) + .some + ) + def downloadZip[F[_]: Sync]( args: DownloadZipArgs, summaryId: Ident, diff --git a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala index a30e0922..fd5f9214 100644 --- a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala +++ b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala @@ -45,7 +45,14 @@ object CreateIndex { chunkSize: Int ): F[Unit] = { val attachs = store - .transact(QAttachment.allAttachmentMetaAndName(collective, itemIds, chunkSize)) + .transact( + QAttachment.allAttachmentMetaAndName( + collective, + itemIds, + ItemState.validStates, + chunkSize + ) + ) .map(caa => TextData .attachment( diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala new file mode 100644 index 00000000..dba1cbf4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import fs2.io.file.Path + +import docspell.addons.AddonExecutorConfig + +final case class AddonEnvConfig( + workingDir: Path, + cacheDir: Path, + executorConfig: AddonExecutorConfig +) diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala new file mode 100644 index 00000000..f69dd2a7 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala @@ -0,0 +1,199 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons._ +import docspell.backend.joex.AddonOps.{AddonRunConfigRef, ExecResult} +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.common.exec.Env +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records.AddonRunConfigResolved + +trait AddonOps[F[_]] { + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + middleware: Middleware[F] + ): F[ExecResult] + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + middleware: Middleware[F] + ): F[ExecResult] + + /** Find enabled addon run config references to be executed. Can be additionally + * filtered by given ids and triggers. + */ + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] + + /** Find enabled addon run config reference given an addon task id */ + def findAddonRef(collective: Ident, runConfigId: Ident): F[Option[AddonRunConfigRef]] + + /** Creates an executor for addons given a configuration. */ + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] + +} + +object AddonOps { + case class AddonRunConfigRef( + id: Ident, + collective: Ident, + userId: Option[Ident], + name: String, + refs: List[AddonRef] + ) + + object AddonRunConfigRef { + def fromResolved(r: AddonRunConfigResolved): AddonRunConfigRef = + AddonRunConfigRef( + r.config.id, + r.config.cid, + r.config.userId, + r.config.name, + r.refs.map(ref => AddonRef(ref.archive.asArchive, ref.ref.args)) + ) + } + + case class ExecResult( + result: List[AddonExecutionResult], + runConfigs: List[AddonRunConfigRef] + ) { + lazy val combined = result.combineAll + } + + object ExecResult { + def runConfigNotFound(id: Ident): ExecResult = + ExecResult( + AddonExecutionResult( + AddonResult.executionFailed( + new Exception(s"Addon run config ${id.id} not found.") + ) :: Nil, + false + ) :: Nil, + Nil + ) + } + + def apply[F[_]: Async]( + cfg: AddonEnvConfig, + store: Store[F], + cmdRunner: BackendCommandRunner[F, Unit], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonOps[F] = + new AddonOps[F] with LoggerExtension { + private[this] val logger = docspell.logging.getLogger[F] + + private val urlReader = FileUrlReader(store.fileRepo) + private val postProcess = AddonPostProcess(cmdRunner, store, attachment, jobStore) + private val prepare = new AddonPrepare[F](store) + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + custom: Middleware[F] + ): F[ExecResult] = + for { + runCfgs <- findAddonRefs(collective, trigger, runConfigIds) + log = logger.getOrElse(this.logger) + _ <- log.info(s"Running ${runCfgs.size} addon tasks for trigger $trigger") + + results <- runCfgs.traverse(r => execRunConfig(log, r, custom)) + } yield ExecResult(results.flatMap(_.result), runCfgs) + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + custom: Middleware[F] + ): F[ExecResult] = + (for { + runCfg <- OptionT(findAddonRef(collective, runConfigId)) + execRes <- OptionT.liftF(execRunConfig(logger, runCfg, custom)) + } yield execRes).getOrElse(ExecResult.runConfigNotFound(runConfigId)) + + def execRunConfig( + logger: Logger[F], + runCfg: AddonRunConfigRef, + custom: Middleware[F] + ): F[ExecResult] = + for { + executor <- getExecutor(cfg.executorConfig) + log = logger.withRunConfig(runCfg) + result <- + Directory.temp(cfg.workingDir, "addon-output-").use { outDir => + val cacheDir = cfg.cacheDir / runCfg.id.id + val inputEnv = + InputEnv(runCfg.refs, cfg.workingDir, outDir, cacheDir, Env.empty) + + for { + middleware <- createMiddleware(custom, runCfg) + res <- middleware(executor.execute(log)).run(inputEnv) + _ <- log.debug(s"Addon result: $res") + _ <- postProcess.onResult(log, runCfg.collective, res, outDir) + } yield res + } + execRes = ExecResult(List(result), List(runCfg)) + } yield execRes + + def createMiddleware(custom: Middleware[F], runCfg: AddonRunConfigRef) = for { + dscMW <- prepare.createDscEnv(runCfg, cfg.executorConfig.runTimeout) + mm = dscMW >> custom >> prepare.logResult(logger, runCfg) >> Middleware + .ephemeralRun[F] + } yield mm + + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] = + Async[F].pure(AddonExecutor(cfg, urlReader)) + + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] = + store + .transact( + AddonRunConfigResolved.findAllForCollective( + collective, + enabled = true.some, + trigger, + runConfigIds + ) + ) + .map(_.map(AddonRunConfigRef.fromResolved)) + + def findAddonRef( + collective: Ident, + runConfigId: Ident + ): F[Option[AddonRunConfigRef]] = + OptionT( + store + .transact( + AddonRunConfigResolved.findById( + runConfigId, + collective, + enabled = Some(true) + ) + ) + ).map(AddonRunConfigRef.fromResolved).value + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala new file mode 100644 index 00000000..2696b4e9 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala @@ -0,0 +1,198 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons._ +import docspell.addons.out.{AddonOutput, ItemFile, NewItem} +import docspell.backend.JobFactory +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.files.FileSupport +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.records._ + +final private[joex] class AddonPostProcess[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachOps: OAttachment[F], + jobStore: JobStore[F] +) extends FileSupport { + + def onResult( + logger: Logger[F], + collective: Ident, + result: AddonExecutionResult, + outputDir: Path + ): F[Unit] = + result.addonResult match { + case AddonResult.Success(output) => + onSuccess(logger, collective, output, outputDir) + case _ => + ().pure[F] + } + + def onSuccess( + logger: Logger[F], + collective: Ident, + output: AddonOutput, + outputDir: Path + ): F[Unit] = + for { + _ <- logger.info("Applying addon output") + _ <- cmdRunner.runAll(collective, output.commands) + _ <- logger.debug("Applying changes from files") + _ <- output.files.traverse_(updateOne(logger, collective, outputDir)) + _ <- output.newItems.traverse_(submitNewItem(logger, collective, outputDir)) + } yield () + + def submitNewItem( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(newItem: NewItem): F[Unit] = + for { + _ <- logger.info(s"Submit new item with ${newItem.files.size} files") + files <- newItem.resolveFiles[F](logger, outputDir) + collLang <- store.transact(RCollective.findLanguage(collective)) + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + _ <- logger.debug(s"Saved ${uploaded.size} files to be processed.") + args = ProcessItemArgs( + newItem.toProcessMeta(collective, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + account = AccountId(collective, DocspellSystem.user) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + + def updateOne(logger: Logger[F], collective: Ident, outputDir: Path)( + itemFile: ItemFile + ): F[Unit] = + for { + textFiles <- itemFile.resolveTextFiles(logger, outputDir) + pdfFiles <- itemFile.resolvePdfFiles(logger, outputDir) + previewFiles <- itemFile.resolvePreviewFiles(logger, outputDir) + attachs <- OptionT + .whenF(textFiles.nonEmpty || pdfFiles.nonEmpty || previewFiles.nonEmpty)( + store.transact(RAttachment.findByItem(itemFile.itemId)) + ) + .getOrElse(Vector.empty) + _ <- textFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + setText(collective, ra, file.readText) + } + } + _ <- pdfFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePdf(collective, ra, file, previewFiles.forall(_._1 != key)) + } + } + _ <- previewFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePreview(collective, ra.id, file) + } + } + _ <- submitNewFiles(logger, collective, outputDir)(itemFile) + } yield () + + def submitNewFiles( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(itemFile: ItemFile): F[Unit] = + for { + _ <- logger.info(s"Submitting new file for item") + collLang <- store.transact(RCollective.findLanguage(collective)) + newFiles <- itemFile.resolveNewFiles(logger, outputDir) + byMeta = newFiles.groupBy(_._1.metadata).view.mapValues(_.map(_._2)) + account = AccountId(collective, DocspellSystem.user) + _ <- byMeta.toList.traverse_ { case (meta, files) => + for { + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + args = ProcessItemArgs( + meta.toProcessMeta(collective, itemFile.itemId, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + } + } yield () + + private def withAttach(logger: Logger[F], key: String, attachs: Vector[RAttachment])( + run: RAttachment => F[Unit] + ): F[Unit] = + OptionT + .fromOption( + attachs.find(a => a.id.id == key || key.toIntOption == a.position.some) + ) + .semiflatMap(run) + .getOrElseF(logger.warn(s"Cannot find attachment for $key to update text!")) + + private def setText(collective: Ident, ra: RAttachment, readText: F[String]): F[Unit] = + attachOps.setExtractedText(collective, ra.itemId, ra.id, readText) + + private def replacePdf( + collective: Ident, + ra: RAttachment, + file: Path, + generatePreview: Boolean + ): F[Unit] = + attachOps.addOrReplacePdf(collective, ra.id, file.readAll, generatePreview) + + private def replacePreview( + collective: Ident, + attachId: Ident, + imageData: Path + ): F[Unit] = + attachOps.addOrReplacePreview(collective, attachId, imageData.readAll) +} + +object AddonPostProcess { + + def apply[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonPostProcess[F] = + new AddonPostProcess[F](cmdRunner, store, attachment, jobStore) +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala new file mode 100644 index 00000000..03ab223d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.auth.AuthToken +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.common._ +import docspell.logging.Logger +import docspell.store.Store +import docspell.store.records.{RNode, RUser} + +import scodec.bits.ByteVector + +private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExtension { + + def logResult(logger: Logger[F], ref: AddonRunConfigRef): Middleware[F] = + Middleware(_.mapF(_.attempt.flatTap { + case Right(_) => ().pure[F] + case Left(ex) => + logger + .withRunConfig(ref) + .warn(ex)(s"Addon task '${ref.id.id}' has failed") + }.rethrow)) + + /** Creates environment variables for dsc to connect to the docspell server for the + * given run config. + */ + def createDscEnv( + runConfigRef: AddonRunConfigRef, + tokenValidity: Duration + ): F[Middleware[F]] = + (for { + userId <- OptionT.fromOption[F](runConfigRef.userId) + user <- OptionT(store.transact(RUser.getIdByIdOrLogin(userId))) + account = AccountId(runConfigRef.collective, user.login) + env = + Middleware.prepare[F]( + Kleisli(input => makeDscEnv(account, tokenValidity).map(input.addEnv)) + ) + } yield env).getOrElse(Middleware.identity[F]) + + /** Creates environment variables to have dsc automatically connect as the given user. + * Additionally a random rest-server is looked up from the database to set its url. + */ + def makeDscEnv( + accountId: AccountId, + tokenValidity: Duration + ): F[Map[String, String]] = + for { + serverNode <- store.transact( + RNode + .findAll(NodeType.Restserver) + .map(_.sortBy(_.updated).lastOption) + ) + url = serverNode.map(_.url).map(u => "DSC_DOCSPELL_URL" -> u.asString) + secret = serverNode.flatMap(_.serverSecret) + + token <- AuthToken.user( + accountId, + false, + secret.getOrElse(ByteVector.empty), + tokenValidity.some + ) + session = ("DSC_SESSION" -> token.asString).some + } yield List(url, session).flatten.toMap +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala new file mode 100644 index 00000000..1f3e4da1 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.logging.Logger + +trait LoggerExtension { + + implicit final class LoggerDataOps[F[_]](self: Logger[F]) { + def withRunConfig(t: AddonRunConfigRef): Logger[F] = + self.capture("addon-task-id", t.id) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala new file mode 100644 index 00000000..e634233a --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} + +sealed trait AddonRunConfigError { + final def cast: AddonRunConfigError = this + + def toLeft[A]: Either[AddonRunConfigError, A] = Left(this) + + def message: String +} + +object AddonRunConfigError { + + case object MissingSchedule extends AddonRunConfigError { + val message = + "The run config has a trigger 'scheduled' but doesn't provide a schedule!" + } + + case object ObsoleteSchedule extends AddonRunConfigError { + val message = "The run config has a schedule, but not a trigger 'Scheduled'." + } + + case class MismatchingTrigger(unsupported: NonEmptyList[(String, AddonTriggerType)]) + extends AddonRunConfigError { + def message: String = { + val list = + unsupported.map { case (name, tt) => s"$name: ${tt.name}" }.toList.mkString(", ") + s"Some listed addons don't support all defined triggers: $list" + } + } + + object MismatchingTrigger { + def apply(addon: AddonMeta, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + + def apply(addon: AddonArchive, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala new file mode 100644 index 00000000..c675c13d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.ops.AddonRunConfigError._ +import docspell.backend.ops.OAddons.{AddonRunConfigResult, AddonRunInsert} +import docspell.common.Ident +import docspell.store.Store +import docspell.store.records.RAddonArchive + +object AddonRunConfigValidate { + + def apply[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[AddonRunInsert]] = { + val init: AddonRunConfigResult[Unit] = ().asRight + + List( + checkScheduled(cfg).pure[F], + checkTriggers(store, cid)(cfg) + ) + .foldLeftM(init)((res, fr) => fr.map(r => res.flatMap(_ => r))) + .map(_.as(cfg)) + } + + def checkTriggers[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[Unit]] = + for { + addons <- store.transact(RAddonArchive.findByIds(cid, cfg.addons.map(_.addonId))) + given = cfg.triggered.toList.toSet + res = addons + .flatMap(r => given.diff(r.triggers).map(tt => r.nameAndVersion -> tt)) + + maybeError = NonEmptyList + .fromList(res) + .map(nel => MismatchingTrigger(nel)) + } yield maybeError.map(_.toLeft).getOrElse(Right(())) + + def checkScheduled(cfg: AddonRunInsert): AddonRunConfigResult[Unit] = + (cfg.isScheduled, cfg.schedule) match { + case (true, None) => MissingSchedule.toLeft[Unit] + case (false, Some(_)) => ObsoleteSchedule.toLeft[Unit] + case _ => ().asRight + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala new file mode 100644 index 00000000..6074f557 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala @@ -0,0 +1,156 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.EitherT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.Path + +import docspell.addons.{AddonMeta, RunnerType} +import docspell.backend.Config +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.{Ident, LenientUri, UrlReader} +import docspell.joexapi.model.AddonSupport +import docspell.store.Store +import docspell.store.records.RAddonArchive + +final class AddonValidate[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + joexOps: OJoex[F] +) { + private[this] val logger = docspell.logging.getLogger[F] + + def fromUrl( + collective: Ident, + url: LenientUri, + reader: UrlReader[F], + localUrl: Option[LenientUri] = None, + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + if (!cfg.enabled) AddonsDisabled.resultF + else if (cfg.isDenied(url)) UrlUntrusted(url).resultF + else if (checkExisting) + store.transact(RAddonArchive.findByUrl(collective, url)).flatMap { + case Some(ar) => + AddonExists("An addon with this url already exists!", ar).resultF + case None => + archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + } + else archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + + def archive( + collective: Ident, + addonData: Either[Path, Stream[F, Byte]], + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + (for { + _ <- EitherT.cond[F](cfg.enabled, (), AddonsDisabled.cast) + + meta <- + EitherT( + addonData + .fold( + AddonMeta.findInDirectory[F], + AddonMeta.findInZip[F] + ) + .attempt + ) + .leftMap(ex => NotAnAddon(ex).cast) + _ <- EitherT.cond( + meta.triggers.exists(_.nonEmpty), + (), + InvalidAddon( + "The addon doesn't define any triggers. At least one is required!" + ).cast + ) + _ <- EitherT.cond( + meta.options.exists(_.isUseful), + (), + InvalidAddon( + "Addon defines no output and no networking. It can't do anything useful." + ).cast + ) + _ <- EitherT.cond(cfg.allowImpure || meta.isPure, (), ImpureAddonsDisabled.cast) + + _ <- + if (checkExisting) + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(collective, meta.meta.name, meta.meta.version) + ) + .map { + case Some(ar) => AddonExists(ar).result + case None => rightUnit + } + ) + else rightUnitT + + joexSupport <- EitherT.liftF(joexOps.getAddonSupport) + addonRunners <- EitherT.liftF(meta.enabledTypes(addonData)) + _ <- EitherT.liftF( + logger.info( + s"Comparing joex support vs addon runner: $joexSupport vs. $addonRunners" + ) + ) + _ <- EitherT.fromEither(validateJoexSupport(addonRunners, joexSupport)) + + } yield meta).value + + private def validateJoexSupport( + addonRunnerTypes: List[RunnerType], + joexSupport: List[AddonSupport] + ): AddonValidationResult[Unit] = { + val addonRunners = addonRunnerTypes.mkString(", ") + for { + _ <- Either.cond( + joexSupport.nonEmpty, + (), + AddonUnsupported("There are no joex nodes that have addons enabled!", Nil).cast + ) + _ <- Either.cond( + addonRunners.nonEmpty, + (), + InvalidAddon("The addon doesn't enable any runner.") + ) + + ids = joexSupport + .map(n => n.nodeId -> n.runners.intersect(addonRunnerTypes).toSet) + + unsupportedJoex = ids.filter(_._2.isEmpty).map(_._1) + + _ <- Either.cond( + ids.forall(_._2.nonEmpty), + (), + AddonUnsupported( + s"A joex node doesn't support this addons runners: $addonRunners. " + + s"Check: ${unsupportedJoex.map(_.id).mkString(", ")}.", + unsupportedJoex + ).cast + ) + } yield () + } + + private def rightUnit: AddonValidationResult[Unit] = + ().asRight[AddonValidationError] + + private def rightUnitT: EitherT[F, AddonValidationError, Unit] = + EitherT.fromEither(rightUnit) + + implicit final class ErrorOps(self: AddonValidationError) { + def result: AddonValidationResult[AddonMeta] = + self.toLeft + + def resultF: F[AddonValidationResult[AddonMeta]] = + result.pure[F] + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala new file mode 100644 index 00000000..d8f77e24 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import docspell.common.{Ident, LenientUri} +import docspell.store.records.RAddonArchive + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AddonValidationError { + def cast: AddonValidationError = this + + def toLeft[A]: Either[AddonValidationError, A] = Left(this) +} + +object AddonValidationError { + + implicit private val throwableDecoder: Decoder[Throwable] = + Decoder.decodeString.map(new Exception(_)) + implicit private val throwableEncoder: Encoder[Throwable] = + Encoder.encodeString.contramap(_.getMessage) + + case object AddonsDisabled extends AddonValidationError {} + + case class UrlUntrusted(url: LenientUri) extends AddonValidationError + object UrlUntrusted { + implicit val jsonDecoder: Decoder[UrlUntrusted] = deriveDecoder + implicit val jsonEncoder: Encoder[UrlUntrusted] = deriveEncoder + } + + case class NotAnAddon(error: Throwable) extends AddonValidationError + object NotAnAddon { + implicit val jsonDecoder: Decoder[NotAnAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[NotAnAddon] = deriveEncoder + } + + case class AddonUnsupported(message: String, affectedNodes: List[Ident]) + extends AddonValidationError + object AddonUnsupported { + implicit val jsonDecoder: Decoder[AddonUnsupported] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonUnsupported] = deriveEncoder + } + + case class InvalidAddon(message: String) extends AddonValidationError + object InvalidAddon { + implicit val jsonDecoder: Decoder[InvalidAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[InvalidAddon] = deriveEncoder + } + + case class AddonExists(message: String, addon: RAddonArchive) + extends AddonValidationError + object AddonExists { + def apply(addon: RAddonArchive): AddonExists = + AddonExists(s"An addon '${addon.name}/${addon.version}' already exists!", addon) + + implicit val jsonDecoder: Decoder[AddonExists] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonExists] = deriveEncoder + } + + case object AddonNotFound extends AddonValidationError + + case class DownloadFailed(error: Throwable) extends AddonValidationError + object DownloadFailed { + implicit val jsonDecoder: Decoder[DownloadFailed] = deriveDecoder + implicit val jsonEncoder: Encoder[DownloadFailed] = deriveEncoder + } + + case object ImpureAddonsDisabled extends AddonValidationError + + case object RefreshLocalAddon extends AddonValidationError + + implicit val jsonConfig: Configuration = + Configuration.default.withKebabCaseConstructorNames + .withDiscriminator("errorType") + + implicit val jsonDecoder: Decoder[AddonValidationError] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonValidationError] = deriveConfiguredEncoder +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala new file mode 100644 index 00000000..be5232a4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala @@ -0,0 +1,426 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{EitherT, NonEmptyList, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.{AddonMeta, AddonTriggerType} +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons._ +import docspell.backend.{Config, JobFactory} +import docspell.common._ +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records._ + +import com.github.eikek.calev.CalEvent + +trait OAddons[F[_]] { + + /** Registers a new addon. An error is returned if an addon with this url already + * exists. + */ + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Refreshes an existing addon by downloading it again and updating metadata. */ + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Look into the addon at the given url and return its metadata. */ + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] + + /** Deletes the addon if it exists. */ + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] + + /** Inserts or updates the addon run configuration. If it already exists (and the given + * id is non empty), it will be completely replaced with the given one. + */ + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] + + /** Deletes this task from the database. */ + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] +} + +object OAddons { + val scheduledAddonTaskName: Ident = + ScheduledAddonTaskArgs.taskName + + case class AddonRunInsert( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: NonEmptyList[AddonTriggerType], + addons: NonEmptyList[AddonArgs] + ) { + + def isScheduled: Boolean = + triggered.exists(_ == AddonTriggerType.Scheduled) + } + case class AddonArgs(addonId: Ident, args: String) + + case class AddonRunInfo( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: List[AddonTriggerType], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + ) + object AddonRunInfo { + def fromRunConfigData( + timer: Option[CalEvent], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + )(t: AddonRunConfigData): AddonRunInfo = + AddonRunInfo( + id = t.runConfig.id, + name = t.runConfig.name, + enabled = t.runConfig.enabled, + userId = t.runConfig.userId, + schedule = timer, + triggered = t.triggers.map(_.trigger), + addons = addons + ) + } + + type AddonRunConfigResult[A] = Either[AddonRunConfigError, A] + object AddonRunConfigResult { + def success[A](value: A): AddonRunConfigResult[A] = Right(value) + def failure[A](error: AddonRunConfigError): AddonRunConfigResult[A] = error.toLeft[A] + } + + type AddonValidationResult[A] = Either[AddonValidationError, A] + object AddonValidationResult { + def success[A](value: A): AddonValidationResult[A] = Right(value) + def failure[A](error: AddonValidationError): AddonValidationResult[A] = Left(error) + } + + def apply[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + userTasks: UserTaskStore[F], + jobStore: JobStore[F], + joex: OJoex[F] + ): OAddons[F] = + new OAddons[F] { + private[this] val logger = docspell.logging.getLogger[F] + private val urlReader = FileUrlReader(store.fileRepo) + private val zip = MimeType.zip.asString + private val addonValidate = new AddonValidate[F](cfg, store, joex) + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] = + for { + all <- store.transact(AddonRunConfigData.findAll(collective)) + runConfigIDs = all.map(_.runConfig.id).toSet + archiveIds = all.flatMap(_.addons.map(_.addonId)).distinct + archives <- NonEmptyList + .fromList(archiveIds) + .fold(List.empty[RAddonArchive].pure[F])(ids => + store.transact(RAddonArchive.findByIds(collective, ids)) + ) + archivesMap = archives.groupBy(_.id) + ptask <- userTasks + .getAll(UserTaskScope.collective(collective)) + .filter(ut => runConfigIDs.contains(ut.id)) + .map(ut => ut.id -> ut) + .compile + .toList + .map(_.toMap) + result = all.map { t => + AddonRunInfo.fromRunConfigData( + ptask.get(t.runConfig.id).map(_.timer), + t.addons.map(raa => (archivesMap(raa.addonId).head, raa)) + )(t) + } + } yield result + + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] = { + val insertDataRaw = AddonRunConfigData( + RAddonRunConfig( + runConfig.id, + collective, + runConfig.userId, + runConfig.name, + runConfig.enabled, + Timestamp.Epoch + ), + runConfig.addons.zipWithIndex.map { case (a, index) => + RAddonRunConfigAddon(Ident.unsafe(""), runConfig.id, a.addonId, a.args, index) + }.toList, + runConfig.triggered + .map(t => RAddonRunConfigTrigger(Ident.unsafe(""), runConfig.id, t)) + .toList + ) + + val upsert = for { + userId <- + OptionT + .fromOption(runConfig.userId) + .flatMapF(uid => store.transact(RUser.getIdByIdOrLogin(uid))) + .map(_.uid) + .value + insertData = + insertDataRaw.copy(runConfig = + insertDataRaw.runConfig.copy(userId = userId.orElse(runConfig.userId)) + ) + id <- + OptionT(store.transact(RAddonRunConfig.findById(collective, runConfig.id))) + .map(rt => + AddonRunConfigData( + rt.copy( + userId = insertData.runConfig.userId, + name = insertData.runConfig.name, + enabled = insertData.runConfig.enabled + ), + insertData.addons, + insertData.triggers + ) + ) + .semiflatMap(rt => + store.transact(AddonRunConfigData.update(rt).as(rt.runConfig.id)) + ) + .getOrElseF(store.transact(AddonRunConfigData.insert(insertData))) + } yield id + + EitherT(AddonRunConfigValidate(store, collective)(runConfig)) + .semiflatMap(_ => + upsert.flatTap { runConfigId => + runConfig.schedule match { + case Some(timer) => + userTasks.updateTask( + UserTaskScope.collective(collective), + s"Addon task ${runConfig.name}".some, + UserTask( + runConfigId, + scheduledAddonTaskName, + true, + timer, + s"Running scheduled addon task ${runConfig.name}".some, + ScheduledAddonTaskArgs(collective, runConfigId) + ) + ) + case None => + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + } + } + ) + .value + } + + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] = { + val deleteRunConfig = + (for { + e <- OptionT(RAddonRunConfig.findById(collective, runConfigId)) + _ <- OptionT.liftF(RAddonRunConfigAddon.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfigTrigger.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfig.deleteById(collective, e.id)) + } yield true).getOrElse(false) + + for { + deleted <- store.transact(deleteRunConfig) + _ <- + if (deleted) + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + else 0.pure[F] + } yield deleted + } + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] = + store.transact(RAddonArchive.listAll(collective)) + + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] = + store.transact(RAddonArchive.deleteById(collective, addonId)).map(_ > 0) + + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] = + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val log = logger.getOrElse(this.logger) + def validateAndInsert(file: FileKey, localUrl: LenientUri) = + addonValidate.fromUrl(collective, url, urlReader, localUrl.some).flatMap { + case Right(meta) => + insertAddon(collective, url, meta, file) + .map(ar => AddonValidationResult.success(ar -> meta)) + + case Left(error) => + store.fileRepo + .delete(file) + .as(AddonValidationResult.failure[(RAddonArchive, AddonMeta)](error)) + } + + log.info(s"Store addon file from '${url.asString} for ${collective.id}") *> + storeAddonFromUrl(collective, url).flatMapF { file => + val localUrl = FileUrlReader.url(file) + for { + _ <- log.info(s"Validating addon…") + res <- validateAndInsert(file, localUrl) + _ <- log.info(s"Validation result: $res") + } yield res + }.value + } + + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val findAddon = store + .transact(RAddonArchive.findById(collective, addonId)) + .map(_.toRight(AddonNotFound)) + def validateAddon(aa: RAddonArchive): F[AddonValidationResult[AddonMeta]] = + aa.originalUrl.fold( + AddonValidationResult.failure[AddonMeta](RefreshLocalAddon).pure[F] + )(url => + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + ) + + EitherT(findAddon).flatMap { aa => + EitherT(validateAddon(aa)) + .flatMap(meta => refreshAddon(aa, meta).map(na => na -> meta)) + }.value + } + + private def refreshAddon( + r: RAddonArchive, + meta: AddonMeta + ): EitherT[F, AddonValidationError, RAddonArchive] = + if (r.isUnchanged(meta)) EitherT.pure(r) + else + r.originalUrl match { + case Some(url) => + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(r.cid, meta.meta.name, meta.meta.version) + ) + .map( + _.fold(().asRight[AddonValidationError])(rx => AddonExists(rx).toLeft) + ) + ).flatMap(_ => + storeAddonFromUrl(r.cid, url).flatMap { file => + val nr = r.update(file, meta) + for { + _ <- EitherT( + store + .transact(RAddonArchive.update(nr)) + .map(_.asRight[AddonValidationError]) + .recoverWith { case ex => + logger.warn(ex)(s"Storing addon metadata failed.") *> + store.fileRepo + .delete(file) + .as( + AddonExists( + s"The addon '${nr.name}/${nr.version}' could not be stored", + nr + ).toLeft + ) + } + ) + _ <- EitherT.liftF(store.fileRepo.delete(r.fileId)) + } yield nr + } + ) + case None => + EitherT.leftT(RefreshLocalAddon.cast) + } + + private def insertAddon( + collective: Ident, + url: LenientUri, + meta: AddonMeta, + file: FileKey + ): F[RAddonArchive] = + for { + now <- Timestamp.current[F] + aId <- Ident.randomId[F] + record = RAddonArchive( + aId, + collective, + file, + url.some, + meta, + now + ) + _ <- store + .transact(RAddonArchive.insert(record, silent = false)) + .onError(_ => store.fileRepo.delete(file)) + } yield record + + private def storeAddonFromUrl(collective: Ident, url: LenientUri) = + for { + urlFile <- EitherT.pure(url.path.segments.lastOption) + file <- EitherT( + urlReader(url) + .through( + store.fileRepo.save( + collective, + FileCategory.Addon, + MimeTypeHint(urlFile, zip.some) + ) + ) + .compile + .lastOrError + .attempt + .map(_.leftMap(DownloadFailed(_).cast)) + ) + } yield file + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] = + for { + jobs <- itemIds.traverse(id => + JobFactory.existingItemAddon( + ItemAddonTaskArgs(account.collective, id, addonRunConfigIds), + account + ) + ) + _ <- jobStore.insertAllIfNew(jobs.map(_.encode).toList) + } yield () + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala new file mode 100644 index 00000000..b5f0ddd8 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala @@ -0,0 +1,223 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{NonEmptyList => Nel, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.backend.JobFactory +import docspell.common.MakePreviewArgs.StoreMode +import docspell.common._ +import docspell.files.TikaMimetype +import docspell.ftsclient.{FtsClient, TextData} +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +trait OAttachment[F[_]] { + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] +} + +object OAttachment { + + def apply[F[_]: Sync]( + store: Store[F], + fts: FtsClient[F], + jobStore: JobStore[F] + ): OAttachment[F] = + new OAttachment[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] = + for { + _ <- logger.info(s"Find attachment ${attachId.id} to update extracted text.") + cca <- store + .transact( + QAttachment + .allAttachmentMetaAndName( + collective.some, + Nel.of(itemId).some, + ItemState.validStates.append(ItemState.Processing), + 100 + ) + ) + .filter(_.id == attachId) + .compile + .last + content = cca.find(_.id == attachId) + _ <- logger.debug(s"Found existing metadata: ${content.isDefined}") + _ <- OptionT + .fromOption(content) + .semiflatMap { cnt => + for { + _ <- logger.debug(s"Setting new extracted text on ${cnt.id.id}") + text <- newText + td = TextData.attachment( + cnt.item, + cnt.id, + cnt.collective, + cnt.folder, + cnt.lang, + cnt.name, + text.some + ) + _ <- store.transact(RAttachmentMeta.updateContent(attachId, text)) + _ <- fts.updateIndex(logger, td) + } yield () + } + .getOrElseF( + logger.warn( + s"Item or attachment meta not found to update text: ${itemId.id}" + ) + ) + } yield () + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] = { + def generatePreview(ra: RAttachment): F[Unit] = + JobFactory + .makePreview(MakePreviewArgs(ra.id, StoreMode.Replace), None) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to re-generate preview from new pdf") + + def generatePageCount(ra: RAttachment): F[Unit] = + JobFactory + .makePageCount( + MakePageCountArgs(ra.id), + AccountId(collective, DocspellSystem.user).some + ) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to find page count from new pdf") + + def setFile(ra: RAttachment, rs: RAttachmentSource) = + for { + _ <- requireMimeType(pdfData, MimeType.pdf) + + newFile <- pdfData + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentConvert, + MimeTypeHint.advertised(MimeType.pdf) + ) + ) + .compile + .lastOrError + + _ <- store.transact(RAttachment.updateFileId(attachId, newFile)) + _ <- logger.info(s"Deleting old file for attachment") + _ <- + if (rs.fileId == ra.fileId) ().pure[F] + else store.fileRepo.delete(ra.fileId) + _ <- + if (regeneratePreview) generatePreview(ra) + else ().pure[F] + _ <- generatePageCount(ra) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + rs <- OptionT( + store.transact(RAttachmentSource.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra, rs)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot replace pdf file. Attachment not found for id: ${attachId.id}" + ) + ) + } + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] = { + def setFile(ra: RAttachment): F[Unit] = + for { + _ <- requireMimeType(imageData, MimeType.image("*")) + newFile <- imageData + .through( + store.fileRepo + .save(collective, FileCategory.PreviewImage, MimeTypeHint.none) + ) + .compile + .lastOrError + + now <- Timestamp.current[F] + record = RAttachmentPreview(ra.id, newFile, None, now) + oldFile <- store.transact(RAttachmentPreview.upsert(record)) + _ <- OptionT + .fromOption(oldFile) + .semiflatMap(store.fileRepo.delete) + .getOrElse(()) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot add/replace preview file. Attachment not found for id: ${attachId.id}" + ) + ) + } + } + + private def requireMimeType[F[_]: Sync]( + data: Stream[F, Byte], + expectedMime: MimeType + ): F[Unit] = + TikaMimetype + .detect(data, MimeTypeHint.advertised(expectedMime)) + .flatMap { mime => + if (expectedMime.matches(mime)) ().pure[F] + else + Sync[F].raiseError( + new IllegalArgumentException( + s"Expected pdf file, but got: ${mime.asString}" + ) + ) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 391a6593..79b799c0 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -61,6 +61,12 @@ trait OItem[F[_]] { collective: Ident ): F[AttachedEvent[UpdateResult]] + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -80,11 +86,13 @@ trait OItem[F[_]] { collective: Ident ): F[UpdateResult] - def setFolder(item: Ident, folder: Option[Ident], collective: Ident): F[UpdateResult] + /** Set or remove the folder on an item. Folder can be the id or name. */ + def setFolder(item: Ident, folder: Option[String], collective: Ident): F[UpdateResult] + /** Set or remove the folder on multiple items. Folder can be the id or name. */ def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] @@ -122,6 +130,13 @@ trait OItem[F[_]] { def setNotes(item: Ident, notes: Option[String], collective: Ident): F[UpdateResult] + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] def setNameMultiple( @@ -288,6 +303,28 @@ object OItem { } } + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] = + if (categories.isEmpty) { + AttachedEvent.only(UpdateResult.success).pure[F] + } else { + val dbtask = + for { + tags <- RTag.findByItem(item) + removeTags = tags.filter(_.category.exists(categories.contains)) + _ <- RTagItem.removeAllTags(item, removeTags.map(_.tagId)) + mkEvent = Event.TagsChanged + .partial(Nel.of(item), Nil, removeTags.map(_.tagId.id).toList) + } yield AttachedEvent(UpdateResult.success)(mkEvent) + + OptionT(store.transact(RItem.checkByIdAndCollective(item, collective))) + .semiflatMap(_ => store.transact(dbtask)) + .getOrElse(AttachedEvent.only(UpdateResult.notFound)) + } + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -420,21 +457,27 @@ object OItem { def setFolder( item: Ident, - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = - UpdateResult - .fromUpdate( - store - .transact(RItem.updateFolder(item, collective, folder)) + for { + result <- store.transact(RItem.updateFolder(item, collective, folder)).attempt + ures = result.fold( + UpdateResult.failure, + t => UpdateResult.fromUpdateRows(t._1) ) - .flatTap( - onSuccessIgnoreError(fts.updateFolder(logger, item, collective, folder)) + _ <- result.fold( + _ => ().pure[F], + t => + onSuccessIgnoreError(fts.updateFolder(logger, item, collective, t._2))( + ures + ) ) + } yield ures def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = for { @@ -615,6 +658,33 @@ object OItem { } ) + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] = + store + .transact(RItem.appendNotes(item, collective, notes, separator)) + .flatMap { + case Some(newNotes) => + store + .transact(RCollective.findLanguage(collective)) + .map(_.getOrElse(Language.English)) + .flatMap(lang => + fts.updateItemNotes(logger, item, collective, lang, newNotes.some) + ) + .attempt + .flatMap { + case Right(()) => ().pure[F] + case Left(ex) => + logger.warn(s"Error updating full-text index: ${ex.getMessage}") + } + .as(UpdateResult.success) + case None => + UpdateResult.notFound.pure[F] + } + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] = UpdateResult .fromUpdate( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index 9f83d46c..bc0b480c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -6,11 +6,13 @@ package docspell.backend.ops -import cats.Applicative import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import fs2.Stream -import docspell.common.Ident +import docspell.common.{Ident, NodeType} +import docspell.joexapi.client.JoexClient +import docspell.joexapi.model.AddonSupport import docspell.pubsub.api.PubSubT import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify} @@ -21,10 +23,16 @@ trait OJoex[F[_]] { def notifyPeriodicTasks: F[Unit] def cancelJob(job: Ident, worker: Ident): F[Unit] + + def getAddonSupport: F[List[AddonSupport]] } object OJoex { - def apply[F[_]: Applicative](pubSub: PubSubT[F]): Resource[F, OJoex[F]] = + def apply[F[_]: Async]( + pubSub: PubSubT[F], + nodes: ONode[F], + joexClient: JoexClient[F] + ): Resource[F, OJoex[F]] = Resource.pure[F, OJoex[F]](new OJoex[F] { def notifyAllNodes: F[Unit] = @@ -35,5 +43,17 @@ object OJoex { def cancelJob(job: Ident, worker: Ident): F[Unit] = pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(()) + + def getAddonSupport: F[List[AddonSupport]] = + for { + joex <- nodes.getNodes(NodeType.Joex) + conc = math.max(2, Runtime.getRuntime.availableProcessors() - 1) + supp <- Stream + .emits(joex) + .covary[F] + .parEvalMap(conc)(n => joexClient.getAddonSupport(n.url)) + .compile + .toList + } yield supp }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala index 8b55ed29..2e729c05 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala @@ -13,11 +13,27 @@ import docspell.common.{Ident, LenientUri, NodeType} import docspell.store.Store import docspell.store.records.RNode +import scodec.bits.ByteVector + trait ONode[F[_]] { - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] def unregister(appId: Ident): F[Unit] + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] + + def getNodes(nodeType: NodeType): F[Vector[RNode]] } object ONode { @@ -25,9 +41,14 @@ object ONode { def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] = Resource.pure[F, ONode[F]](new ONode[F] { val logger = docspell.logging.getLogger[F] - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] = for { - node <- RNode(appId, nodeType, uri) + node <- RNode(appId, nodeType, uri, serverSecret) _ <- logger.info(s"Registering node ${node.id.id}") _ <- store.transact(RNode.set(node)) } yield () @@ -35,6 +56,19 @@ object ONode { def unregister(appId: Ident): F[Unit] = logger.info(s"Unregister app ${appId.id}") *> store.transact(RNode.delete(appId)).map(_ => ()) + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] = + Resource.make(register(appId, nodeType, uri, serverSecret))(_ => + unregister(appId) + ) + + def getNodes(nodeType: NodeType): F[Vector[RNode]] = + store.transact(RNode.findAll(nodeType)) }) } diff --git a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala index 00453680..fdf77e2b 100644 --- a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala +++ b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala @@ -9,8 +9,9 @@ package docspell.common import java.time.Instant import io.circe._ +import scodec.bits.ByteVector -object BaseJsonCodecs { +trait BaseJsonCodecs { implicit val encodeInstantEpoch: Encoder[Instant] = Encoder.encodeJavaLong.contramap(_.toEpochMilli) @@ -18,4 +19,11 @@ object BaseJsonCodecs { implicit val decodeInstantEpoch: Decoder[Instant] = Decoder.decodeLong.map(Instant.ofEpochMilli) + implicit val byteVectorEncoder: Encoder[ByteVector] = + Encoder.encodeString.contramap(_.toBase64) + + implicit val byteVectorDecoder: Decoder[ByteVector] = + Decoder.decodeString.emap(ByteVector.fromBase64Descriptive(_)) } + +object BaseJsonCodecs extends BaseJsonCodecs diff --git a/modules/common/src/main/scala/docspell/common/Binary.scala b/modules/common/src/main/scala/docspell/common/Binary.scala index 2dc3c354..fa36ed1c 100644 --- a/modules/common/src/main/scala/docspell/common/Binary.scala +++ b/modules/common/src/main/scala/docspell/common/Binary.scala @@ -18,6 +18,18 @@ final case class Binary[F[_]](name: String, mime: MimeType, data: Stream[F, Byte def withMime(mime: MimeType): Binary[F] = copy(mime = mime) + + /** Return the extension of `name` if available (without the dot) */ + def extension: Option[String] = + name.lastIndexOf('.') match { + case n if n > 0 => + Some(name.substring(n + 1)) + case _ => + None + } + + def extensionIn(extensions: Set[String]): Boolean = + extension.exists(extensions.contains) } object Binary { diff --git a/modules/common/src/main/scala/docspell/common/FileCategory.scala b/modules/common/src/main/scala/docspell/common/FileCategory.scala index cdf6c723..21caa25c 100644 --- a/modules/common/src/main/scala/docspell/common/FileCategory.scala +++ b/modules/common/src/main/scala/docspell/common/FileCategory.scala @@ -32,6 +32,7 @@ object FileCategory { case object PreviewImage extends FileCategory case object Classifier extends FileCategory case object DownloadAll extends FileCategory + case object Addon extends FileCategory val all: NonEmptyList[FileCategory] = NonEmptyList.of( @@ -39,7 +40,8 @@ object FileCategory { AttachmentConvert, PreviewImage, Classifier, - DownloadAll + DownloadAll, + Addon ) def fromString(str: String): Either[String, FileCategory] = diff --git a/modules/common/src/main/scala/docspell/common/Glob.scala b/modules/common/src/main/scala/docspell/common/Glob.scala index 722af9fd..5b7ee9ca 100644 --- a/modules/common/src/main/scala/docspell/common/Glob.scala +++ b/modules/common/src/main/scala/docspell/common/Glob.scala @@ -32,7 +32,8 @@ object Glob { def single(str: String) = PatternGlob(Pattern(split(str, separator).map(makeSegment))) - if (in == "*") all + if (in == all.asString) all + else if (in == none.asString) none else split(in, anyChar) match { case NonEmptyList(_, Nil) => @@ -51,15 +52,25 @@ object Glob { val asString = "*" } + val none = new Glob { + def matches(caseSensitive: Boolean)(in: String) = false + def matchFilenameOrPath(in: String) = false + def asString = "!*" + } + def pattern(pattern: Pattern): Glob = PatternGlob(pattern) /** A simple glob supporting `*` and `?`. */ final private case class PatternGlob(pattern: Pattern) extends Glob { - def matches(caseSensitive: Boolean)(in: String): Boolean = + def matches(caseSensitive: Boolean)(in: String): Boolean = { + val input = Glob.split(in, Glob.separator) + + pattern.parts.size == input.size && pattern.parts - .zipWith(Glob.split(in, Glob.separator))(_.matches(caseSensitive)(_)) + .zipWith(input)(_.matches(caseSensitive)(_)) .forall(identity) + } def matchFilenameOrPath(in: String): Boolean = if (pattern.parts.tail.isEmpty) matches(true)(split(in, separator).last) @@ -67,6 +78,8 @@ object Glob { def asString: String = pattern.asString + + override def toString = s"PatternGlob($asString)" } final private case class AnyGlob(globs: NonEmptyList[Glob]) extends Glob { @@ -76,6 +89,8 @@ object Glob { globs.exists(_.matchFilenameOrPath(in)) def asString = globs.toList.map(_.asString).mkString(anyChar.toString) + + override def toString = s"AnyGlob($globs)" } case class Pattern(parts: NonEmptyList[Segment]) { diff --git a/modules/common/src/main/scala/docspell/common/Ident.scala b/modules/common/src/main/scala/docspell/common/Ident.scala index 2f630e33..928f0dd5 100644 --- a/modules/common/src/main/scala/docspell/common/Ident.scala +++ b/modules/common/src/main/scala/docspell/common/Ident.scala @@ -26,6 +26,9 @@ case class Ident(id: String) { def /(next: Ident): Ident = new Ident(id + Ident.concatChar + next.id) + + def take(n: Int): Ident = + new Ident(id.take(n)) } object Ident { diff --git a/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala new file mode 100644 index 00000000..3333486d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +/** Arguments to submit a task that runs addons configured for some existing item. + * + * If `addonTaskIds` is non empty, only these addon tasks are run. Otherwise all addon + * tasks that are configured for 'existing-item' are run. + */ +final case class ItemAddonTaskArgs( + collective: Ident, + itemId: Ident, + addonRunConfigs: Set[Ident] +) + +object ItemAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-existing-item") + + implicit val jsonDecoder: Decoder[ItemAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala index 55fbbda1..72da5281 100644 --- a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala +++ b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala @@ -6,6 +6,8 @@ package docspell.common +import fs2.io.file.Path + case class MimeTypeHint(filename: Option[String], advertised: Option[String]) { def withName(name: String): MimeTypeHint = @@ -21,6 +23,9 @@ object MimeTypeHint { def filename(name: String): MimeTypeHint = MimeTypeHint(Some(name), None) + def filename(file: Path): MimeTypeHint = + filename(file.fileName.toString) + def advertised(mimeType: MimeType): MimeTypeHint = advertised(mimeType.asString) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index 8398eaf0..9a92cee7 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -17,7 +17,7 @@ import io.circe.generic.semiauto._ * This task is run for each new file to create a new item from it or to add this file as * an attachment to an existing item. * - * If the `itemId' is set to some value, the item is tried to load to ammend with the + * If the `itemId' is set to some value, the item is tried to load to amend with the * given files. Otherwise a new item is created. * * It is also re-used by the 'ReProcessItem' task. diff --git a/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala new file mode 100644 index 00000000..5104108e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class ScheduledAddonTaskArgs(collective: Ident, addonTaskId: Ident) + +object ScheduledAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-scheduled-task") + + implicit val jsonDecoder: Decoder[ScheduledAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ScheduledAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/SystemCommand.scala b/modules/common/src/main/scala/docspell/common/SystemCommand.scala index 63f4ca40..702c546d 100644 --- a/modules/common/src/main/scala/docspell/common/SystemCommand.scala +++ b/modules/common/src/main/scala/docspell/common/SystemCommand.scala @@ -17,11 +17,23 @@ import cats.implicits._ import fs2.io.file.Path import fs2.{Stream, io, text} +import docspell.common.{exec => newExec} import docspell.logging.Logger +// better use `SysCmd` and `SysExec` object SystemCommand { - final case class Config(program: String, args: Seq[String], timeout: Duration) { + final case class Config( + program: String, + args: Seq[String], + timeout: Duration, + env: Map[String, String] = Map.empty + ) { + + def toSysCmd = newExec + .SysCmd(program, newExec.Args(args)) + .withTimeout(timeout) + .addEnv(newExec.Env(env)) def mapArgs(f: String => String): Config = Config(program, args.map(f), timeout) @@ -33,6 +45,18 @@ object SystemCommand { } ) + def withEnv(key: String, value: String): Config = + copy(env = env.updated(key, value)) + + def addEnv(moreEnv: Map[String, String]): Config = + copy(env = env ++ moreEnv) + + def appendArgs(extraArgs: Args): Config = + copy(args = args ++ extraArgs.args) + + def appendArgs(extraArgs: Seq[String]): Config = + copy(args = args ++ extraArgs) + def toCmd: List[String] = program :: args.toList @@ -40,6 +64,45 @@ object SystemCommand { toCmd.mkString(" ") } + final case class Args(args: Vector[String]) extends Iterable[String] { + override def iterator = args.iterator + + def prepend(a: String): Args = Args(a +: args) + + def prependWhen(flag: Boolean)(a: String): Args = + prependOption(Option.when(flag)(a)) + + def prependOption(value: Option[String]): Args = + value.map(prepend).getOrElse(this) + + def append(a: String, as: String*): Args = + Args(args ++ (a +: as.toVector)) + + def appendOption(value: Option[String]): Args = + value.map(append(_)).getOrElse(this) + + def appendOptionVal(first: String, second: Option[String]): Args = + second.map(b => append(first, b)).getOrElse(this) + + def appendWhen(flag: Boolean)(a: String, as: String*): Args = + if (flag) append(a, as: _*) else this + + def appendWhenNot(flag: Boolean)(a: String, as: String*): Args = + if (!flag) append(a, as: _*) else this + + def append(p: Path): Args = + append(p.toString) + + def append(as: Iterable[String]): Args = + Args(args ++ as.toVector) + } + object Args { + val empty: Args = Args() + + def apply(as: String*): Args = + Args(as.toVector) + } + final case class Result(rc: Int, stdout: String, stderr: String) def exec[F[_]: Sync]( @@ -104,6 +167,10 @@ object SystemCommand { .redirectError(Redirect.PIPE) .redirectOutput(Redirect.PIPE) + val pbEnv = pb.environment() + cmd.env.foreach { case (key, value) => + pbEnv.put(key, value) + } wd.map(_.toNioPath.toFile).foreach(pb.directory) pb.start() } diff --git a/modules/common/src/main/scala/docspell/common/UrlMatcher.scala b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala new file mode 100644 index 00000000..c8fd393e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList +import cats.kernel.Monoid +import cats.syntax.all._ + +trait UrlMatcher { + def matches(url: LenientUri): Boolean +} + +object UrlMatcher { + val True = instance(_ => true) + val False = instance(_ => false) + + def instance(f: LenientUri => Boolean): UrlMatcher = + (url: LenientUri) => f(url) + + def fromString(str: String): Either[String, UrlMatcher] = + if (str == "") False.asRight + else if (str == "*") True.asRight + else LenientUri.parse(str).map(fromUrl) + + def unsafeFromString(str: String): UrlMatcher = + fromString(str).fold(sys.error, identity) + + def fromStringList(str: List[String]): Either[String, UrlMatcher] = + str match { + case Nil => False.asRight + case _ => str.map(_.trim).traverse(fromString).map(_.combineAll) + } + + def fromUrl(url: LenientUri): UrlMatcher = { + val schemeGlob = Glob(url.scheme.head) + val hostGlob = HostGlob(url.host) + val pathGlob = Glob(url.path.asString) + new Impl(schemeGlob, hostGlob, pathGlob, url.path.segments.size) + } + + def any(ulrm: IterableOnce[UrlMatcher]): UrlMatcher = + anyMonoid.combineAll(ulrm) + + def all(urlm: IterableOnce[UrlMatcher]): UrlMatcher = + allMonoid.combineAll(urlm) + + val anyMonoid: Monoid[UrlMatcher] = + Monoid.instance(False, (a, b) => instance(url => a.matches(url) || b.matches(url))) + + val allMonoid: Monoid[UrlMatcher] = + Monoid.instance(True, (a, b) => instance(url => a.matches(url) && b.matches(url))) + + implicit val defaultMonoid: Monoid[UrlMatcher] = anyMonoid + + private class Impl(scheme: Glob, host: HostGlob, path: Glob, pathSegmentCount: Int) + extends UrlMatcher { + def matches(url: LenientUri) = { + // strip path to only match prefixes + val mPath: LenientUri.Path = + NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match { + case Some(nel) => LenientUri.NonEmptyPath(nel) + case None => LenientUri.RootPath + } + + url.scheme.forall(scheme.matches(false)) && + host.matches(url.host) && + path.matchFilenameOrPath(mPath.asString) + } + } + + private class HostGlob(glob: Option[Glob]) { + def matches(host: Option[String]): Boolean = + (glob, host) match { + case (Some(pattern), Some(word)) => + pattern.matches(false)(HostGlob.prepareHost(word)) + case (None, None) => true + case _ => false + } + + override def toString = s"HostGlob(${glob.map(_.asString)})" + } + + private object HostGlob { + def apply(hostPattern: Option[String]): HostGlob = + new HostGlob(hostPattern.map(p => Glob(prepareHost(p)))) + + private def prepareHost(host: String): String = + host.replace('.', '/') + } +} diff --git a/modules/common/src/main/scala/docspell/common/UrlReader.scala b/modules/common/src/main/scala/docspell/common/UrlReader.scala new file mode 100644 index 00000000..7e1521f2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlReader.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.ApplicativeError +import cats.effect._ +import fs2.Stream + +trait UrlReader[F[_]] { + def apply(url: LenientUri): Stream[F, Byte] +} + +object UrlReader { + + def instance[F[_]](f: LenientUri => Stream[F, Byte]): UrlReader[F] = + (url: LenientUri) => f(url) + + def failWith[F[_]]( + message: String + )(implicit F: ApplicativeError[F, Throwable]): UrlReader[F] = + instance(url => + Stream.raiseError( + new IllegalStateException(s"Unable to read '${url.asString}': $message") + ) + ) + + def apply[F[_]](implicit r: UrlReader[F]): UrlReader[F] = r + + implicit def defaultReader[F[_]: Sync]: UrlReader[F] = + instance(_.readURL[F](8192)) +} diff --git a/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala new file mode 100644 index 00000000..f0caa3a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AttachmentAction {} + +object AttachmentAction { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class SetExtractedText(text: Option[String]) extends AttachmentAction + object SetExtractedText { + implicit val jsonDecoder: Decoder[SetExtractedText] = deriveDecoder + implicit val jsonEncoder: Encoder[SetExtractedText] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[AttachmentAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AttachmentAction] = deriveConfiguredEncoder + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala new file mode 100644 index 00000000..f5732f64 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait BackendCommand {} + +object BackendCommand { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("command").withKebabCaseConstructorNames + + case class ItemUpdate(itemId: Ident, actions: List[ItemAction]) extends BackendCommand + object ItemUpdate { + implicit val jsonDecoder: Decoder[ItemUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemUpdate] = deriveEncoder + } + + def item(itemId: Ident, actions: List[ItemAction]): BackendCommand = + ItemUpdate(itemId, actions) + + case class AttachmentUpdate( + itemId: Ident, + attachId: Ident, + actions: List[AttachmentAction] + ) extends BackendCommand + object AttachmentUpdate { + implicit val jsonDecoder: Decoder[AttachmentUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachmentUpdate] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[BackendCommand] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[BackendCommand] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala new file mode 100644 index 00000000..863e35ed --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +trait BackendCommandRunner[F[_], A] { + + def run(collective: Ident, cmd: BackendCommand): F[A] + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[A] + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala new file mode 100644 index 00000000..16a09048 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait ItemAction {} + +object ItemAction { + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class AddTags(tags: Set[String]) extends ItemAction + object AddTags { + implicit val jsonDecoder: Decoder[AddTags] = deriveDecoder + implicit val jsonEncoder: Encoder[AddTags] = deriveEncoder + } + + case class ReplaceTags(tags: Set[String]) extends ItemAction + object ReplaceTags { + implicit val jsonDecoder: Decoder[ReplaceTags] = deriveDecoder + implicit val jsonEncoder: Encoder[ReplaceTags] = deriveEncoder + } + + case class RemoveTags(tags: Set[String]) extends ItemAction + object RemoveTags { + implicit val jsonDecoder: Decoder[RemoveTags] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTags] = deriveEncoder + } + + case class RemoveTagsCategory(categories: Set[String]) extends ItemAction + object RemoveTagsCategory { + implicit val jsonDecoder: Decoder[RemoveTagsCategory] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTagsCategory] = deriveEncoder + } + + case class SetFolder(folder: Option[String]) extends ItemAction + object SetFolder { + implicit val jsonDecoder: Decoder[SetFolder] = deriveDecoder + implicit val jsonEncoder: Encoder[SetFolder] = deriveEncoder + } + + case class SetCorrOrg(id: Option[Ident]) extends ItemAction + object SetCorrOrg { + implicit val jsonDecoder: Decoder[SetCorrOrg] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrOrg] = deriveEncoder + } + + case class SetCorrPerson(id: Option[Ident]) extends ItemAction + object SetCorrPerson { + implicit val jsonDecoder: Decoder[SetCorrPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrPerson] = deriveEncoder + } + + case class SetConcPerson(id: Option[Ident]) extends ItemAction + object SetConcPerson { + implicit val jsonDecoder: Decoder[SetConcPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcPerson] = deriveEncoder + } + + case class SetConcEquipment(id: Option[Ident]) extends ItemAction + object SetConcEquipment { + implicit val jsonDecoder: Decoder[SetConcEquipment] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcEquipment] = deriveEncoder + } + + case class SetField(field: Ident, value: String) extends ItemAction + object SetField { + implicit val jsonDecoder: Decoder[SetField] = deriveDecoder + implicit val jsonEncoder: Encoder[SetField] = deriveEncoder + } + + case class SetName(name: String) extends ItemAction + object SetName { + implicit val jsonDecoder: Decoder[SetName] = deriveDecoder + implicit val jsonEncoder: Encoder[SetName] = deriveEncoder + } + + case class SetNotes(notes: Option[String]) extends ItemAction + object SetNotes { + implicit val jsonDecoder: Decoder[SetNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[SetNotes] = deriveEncoder + } + + case class AddNotes(notes: String, separator: Option[String]) extends ItemAction + object AddNotes { + implicit val jsonDecoder: Decoder[AddNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[AddNotes] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[ItemAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[ItemAction] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Args.scala b/modules/common/src/main/scala/docspell/common/exec/Args.scala new file mode 100644 index 00000000..292899a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Args.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import fs2.io.file.Path + +case class Args(values: Seq[String]) { + + def option(key: String, value: String): Args = + Args(values ++ Seq(key, value)) + + def option(key: String, value: Option[String]): Args = + value.map(v => option(key, v)).getOrElse(this) + + def appendOpt(v: Option[String]): Args = + v.map(e => Args(values :+ e)).getOrElse(this) + + def append(v: String, vs: String*): Args = + Args(values ++ (v +: vs)) + + def append(path: Path): Args = + append(path.toString) + + def append(args: Args): Args = + Args(values ++ args.values) + + def append(args: Seq[String]): Args = + Args(values ++ args) + + def prepend(v: String): Args = + Args(v +: values) + + def prependWhen(flag: Boolean)(v: String) = + if (flag) prepend(v) else this + + def cmdString: String = + values.mkString(" ") +} + +object Args { + val empty: Args = Args(Seq.empty) + + def of(v: String*): Args = + Args(v) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Env.scala b/modules/common/src/main/scala/docspell/common/exec/Env.scala new file mode 100644 index 00000000..2524d35a --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Env.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +case class Env(values: Map[String, String]) { + + def add(name: String, value: String): Env = + copy(values.updated(name, value)) + + def addAll(v: Map[String, String]): Env = + Env(values ++ v) + + def addAll(e: Env): Env = + Env(values ++ e.values) + + def ++(e: Env) = addAll(e) + + def foreach(f: (String, String) => Unit): Unit = + values.foreach(t => f(t._1, t._2)) + + def map[A](f: (String, String) => A): Seq[A] = + values.map(f.tupled).toSeq + + def mapConcat[A](f: (String, String) => Seq[A]): Seq[A] = + values.flatMap(f.tupled).toSeq +} + +object Env { + val empty: Env = Env(Map.empty) + + def of(nv: (String, String)*): Env = + Env(Map(nv: _*)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala new file mode 100644 index 00000000..b94b8c4b --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import docspell.common._ + +final case class SysCmd( + program: String, + args: Args, + env: Env, + timeout: Duration +) { + + def withArgs(f: Args => Args): SysCmd = + copy(args = f(args)) + + def withTimeout(to: Duration): SysCmd = + copy(timeout = to) + + def withEnv(f: Env => Env): SysCmd = + copy(env = f(env)) + + def addEnv(env: Env): SysCmd = + withEnv(_.addAll(env)) + + def cmdString: String = + s"$program ${args.cmdString}" + + private[exec] def toCmd: Seq[String] = + program +: args.values +} + +object SysCmd { + def apply(prg: String, args: String*): SysCmd = + apply(prg, Args(args)) + + def apply(prg: String, args: Args): SysCmd = + SysCmd(prg, args, Env.empty, Duration.minutes(2)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysExec.scala b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala new file mode 100644 index 00000000..da7b10c2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala @@ -0,0 +1,163 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import java.lang.ProcessBuilder.Redirect +import java.util.concurrent.TimeUnit + +import scala.concurrent.TimeoutException +import scala.jdk.CollectionConverters._ + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Path +import fs2.{Pipe, Stream} + +import docspell.common.Duration +import docspell.logging.Logger + +trait SysExec[F[_]] { + + def stdout: Stream[F, Byte] + + def stdoutLines: Stream[F, String] = + stdout + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def stderr: Stream[F, Byte] + + def stderrLines: Stream[F, String] = + stderr + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def waitFor(timeout: Option[Duration] = None): F[Int] + + /** Sends a signal to the process to terminate it immediately */ + def cancel: F[Unit] + + /** Consume lines of output of the process in background. */ + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] + + /** Consumes stderr lines (left) and stdout lines (right) in a background thread. */ + def consumeOutputs( + m: Either[String, String] => F[Unit] + )(implicit F: Async[F]): Resource[F, SysExec[F]] = { + val pe: Pipe[F, String, Unit] = _.map(_.asLeft).evalMap(m) + val po: Pipe[F, String, Unit] = _.map(_.asRight).evalMap(m) + consumeOutputs(po, pe) + } + + def logOutputs(logger: Logger[F], name: String)(implicit F: Async[F]) = + consumeOutputs { + case Right(line) => logger.debug(s"[$name (out)]: $line") + case Left(line) => logger.debug(s"[$name (err)]: $line") + } +} + +object SysExec { + private val readChunkSz = 8 * 1024 + + def apply[F[_]: Sync]( + cmd: SysCmd, + logger: Logger[F], + workdir: Option[Path] = None, + stdin: Option[Stream[F, Byte]] = None + ): Resource[F, SysExec[F]] = + for { + proc <- startProcess(logger, cmd, workdir, stdin) + fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil)) + } yield new SysExec[F] { + def stdout: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getInputStream), + readChunkSz, + closeAfterUse = false + ) + + def stderr: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getErrorStream), + readChunkSz, + closeAfterUse = false + ) + + def cancel = Sync[F].blocking(proc.destroy()) + + def waitFor(timeout: Option[Duration]): F[Int] = { + val to = timeout.getOrElse(cmd.timeout) + logger.trace("Waiting for command to terminate…") *> + Sync[F] + .blocking(proc.waitFor(to.millis, TimeUnit.MILLISECONDS)) + .flatTap(_ => fibers.get.flatMap(_.traverse_(identity))) + .flatMap(terminated => + if (terminated) proc.exitValue().pure[F] + else + Sync[F] + .raiseError( + new TimeoutException(s"Timed out after: ${to.formatExact}") + ) + ) + } + + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] = + for { + f1 <- F.background(stdoutLines.through(out).compile.drain) + f2 <- F.background(stderrLines.through(err).compile.drain) + _ <- Resource.eval(fibers.update(list => f1.void :: f2.void :: list)) + } yield this + } + + private def startProcess[F[_]: Sync, A]( + logger: Logger[F], + cmd: SysCmd, + workdir: Option[Path], + stdin: Option[Stream[F, Byte]] + ): Resource[F, Process] = { + val log = logger.debug(s"Running external command: ${cmd.cmdString}") + + val proc = log *> + Sync[F].blocking { + val pb = new ProcessBuilder(cmd.toCmd.asJava) + .redirectInput(if (stdin.isDefined) Redirect.PIPE else Redirect.INHERIT) + .redirectError(Redirect.PIPE) + .redirectOutput(Redirect.PIPE) + + val pbEnv = pb.environment() + cmd.env.foreach { (name, v) => + pbEnv.put(name, v) + () + } + workdir.map(_.toNioPath.toFile).foreach(pb.directory) + pb.start() + } + + Resource + .make(proc)(p => + logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy()) + ) + .evalMap(p => + stdin match { + case Some(in) => + writeToProcess(in, p).compile.drain.as(p) + case None => + p.pure[F] + } + ) + } + + private def writeToProcess[F[_]: Sync]( + data: Stream[F, Byte], + proc: Process + ): Stream[F, Nothing] = + data.through(fs2.io.writeOutputStream(Sync[F].blocking(proc.getOutputStream))) +} diff --git a/modules/common/src/main/scala/docspell/common/File.scala b/modules/common/src/main/scala/docspell/common/util/File.scala similarity index 91% rename from modules/common/src/main/scala/docspell/common/File.scala rename to modules/common/src/main/scala/docspell/common/util/File.scala index 0d6505dd..679c99ec 100644 --- a/modules/common/src/main/scala/docspell/common/File.scala +++ b/modules/common/src/main/scala/docspell/common/util/File.scala @@ -4,20 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.common +package docspell.common.util import java.nio.file.{Path => JPath} -import cats.FlatMap -import cats.Monad import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import cats.{FlatMap, Monad} import fs2.Stream import fs2.io.file.{Files, Flags, Path} -import docspell.common.syntax.all._ - import io.circe.Decoder +import io.circe.parser object File { @@ -75,6 +73,5 @@ object File { .map(_ => file) def readJson[F[_]: Async, A](file: Path)(implicit d: Decoder[A]): F[A] = - readText[F](file).map(_.parseJsonAs[A]).rethrow - + readText[F](file).map(parser.decode[A]).rethrow } diff --git a/modules/common/src/main/scala/docspell/common/util/Random.scala b/modules/common/src/main/scala/docspell/common/util/Random.scala new file mode 100644 index 00000000..b192aa6d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/util/Random.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.util + +import cats.effect._ + +import scodec.bits.ByteVector + +trait Random[F[_]] { + def string(len: Int): F[String] + def string: F[String] = string(8) +} + +object Random { + def apply[F[_]: Sync] = + new Random[F] { + def string(len: Int) = Sync[F].delay { + val buf = Array.ofDim[Byte](len) + new scala.util.Random().nextBytes(buf) + ByteVector.view(buf).toBase58 + } + } +} diff --git a/modules/common/src/test/scala/docspell/common/GlobTest.scala b/modules/common/src/test/scala/docspell/common/GlobTest.scala index 7e3d3bf8..4ee7d468 100644 --- a/modules/common/src/test/scala/docspell/common/GlobTest.scala +++ b/modules/common/src/test/scala/docspell/common/GlobTest.scala @@ -70,11 +70,13 @@ class GlobTest extends FunSuite { test("with splitting") { assert(Glob("a/b/*").matches(true)("a/b/hello")) + assert(!Glob("a/b/*").matches(true)("a/b/hello/bello")) assert(!Glob("a/b/*").matches(true)("/a/b/hello")) assert(Glob("/a/b/*").matches(true)("/a/b/hello")) assert(!Glob("/a/b/*").matches(true)("a/b/hello")) assert(!Glob("*/a/b/*").matches(true)("a/b/hello")) assert(Glob("*/a/b/*").matches(true)("test/a/b/hello")) + assert(!Glob("/a/b").matches(true)("/a/b/c/d")) } test("asString") { diff --git a/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala new file mode 100644 index 00000000..51fb1f19 --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import munit._ + +class UrlMatcherTest extends FunSuite { + + test("it should match patterns") { + assertUrlsMatch( + uri("https://github.com/docspell/*") -> uri("https://github.com/docspell/dsc"), + uri("*s://test.com/*") -> uri("https://test.com/a"), + uri("*s://test.com/*") -> uri("https://test.com/a/b"), + uri("*s://test.com/*") -> uri("https://test.com/a/b/c"), + uri("*s://test.com/project/*") -> uri("https://test.com/project/c"), + uri("https://*.test.com/projects/*") -> uri("https://a.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1") + ) + + assertUrlsNotMatch( + uri("https://*.test.com/projects/*") -> uri("https://test.com/projects/p1"), + uri("*s://test.com/project/*") -> uri("https://test.com/subject/c") + ) + } + + def uri(str: String): LenientUri = LenientUri.unsafe(str) + + def assertUrlsMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri does not match $checkUri" + ) + } + + def assertUrlsMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsMatch(test :: more.toList) + + def assertUrlsNotMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + !UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri incorrectly matches $checkUri" + ) + } + + def assertUrlsNotMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsNotMatch(test :: more.toList) +} diff --git a/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala new file mode 100644 index 00000000..bb0cfaef --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common._ + +import io.circe.parser +import io.circe.syntax._ +import munit._ + +class BackendCommandTest extends FunSuite { + + test("encode json") { + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals( + bc.asJson.spaces2, + """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + ) + } + + test("decode case insensitive keys") { + val json = """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals(parser.decode[BackendCommand](json), Right(bc)) + } + + def id(str: String) = Ident.unsafe(str) +} diff --git a/modules/config/src/main/scala/docspell/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala index e529a2c5..014643ae 100644 --- a/modules/config/src/main/scala/docspell/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -13,6 +13,7 @@ import scala.reflect.ClassTag import cats.syntax.all._ import fs2.io.file.Path +import docspell.addons.RunnerType import docspell.common._ import docspell.ftspsql.{PgQueryParser, RankNormalization} import docspell.logging.{Level, LogConfig} @@ -32,6 +33,17 @@ object Implicits { else super.fieldValue(name) } + implicit val urlMatcherReader: ConfigReader[UrlMatcher] = { + val fromList = ConfigReader[List[String]].emap(reason(UrlMatcher.fromStringList)) + val fromString = ConfigReader[String].emap( + reason(str => UrlMatcher.fromStringList(str.split("[\\s,]+").toList)) + ) + fromList.orElse(fromString) + } + + implicit val runnerSelectReader: ConfigReader[List[RunnerType]] = + ConfigReader[String].emap(reason(RunnerType.fromSeparatedString)) + implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) diff --git a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala index eff5a0fb..8f0f9e11 100644 --- a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala +++ b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala @@ -12,6 +12,7 @@ import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt} import docspell.logging.Logger diff --git a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala index 25905afe..06ee81c1 100644 --- a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala @@ -15,6 +15,7 @@ import cats.implicits._ import fs2.Stream import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.convert.extern.OcrMyPdfConfig import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig} diff --git a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala index 96f251ff..ad41c01c 100644 --- a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala +++ b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala @@ -18,6 +18,7 @@ import fs2.io.file.Path import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.files.TikaMimetype diff --git a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala index 6f0ab2ab..9beaed28 100644 --- a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala @@ -14,6 +14,7 @@ import cats.effect.unsafe.implicits.global import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.convert._ import docspell.files.ExampleFiles import docspell.logging.TestLoggingConfig diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala index 727666a8..f39e4b0b 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala @@ -11,6 +11,7 @@ import fs2.Stream import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger object Ocr { diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala index 003fa8c3..856c21a3 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala @@ -11,6 +11,7 @@ import java.nio.file.Paths import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File case class OcrConfig( maxImageSize: Int, diff --git a/modules/files/src/main/scala/docspell/files/FileSupport.scala b/modules/files/src/main/scala/docspell/files/FileSupport.scala new file mode 100644 index 00000000..5219ac37 --- /dev/null +++ b/modules/files/src/main/scala/docspell/files/FileSupport.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.files + +import cats.data.OptionT +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.{MimeType, MimeTypeHint} + +import io.circe.Encoder +import io.circe.syntax._ + +trait FileSupport { + implicit final class FileOps[F[_]: Files: Sync](self: Path) { + def detectMime: F[Option[MimeType]] = + Files[F].isReadable(self).flatMap { flag => + OptionT + .whenF(flag) { + TikaMimetype + .detect( + Files[F].readAll(self), + MimeTypeHint.filename(self.fileName.toString) + ) + } + .value + } + + def asTextFile(alt: MimeType => F[Unit]): F[Option[Path]] = + OptionT(detectMime).flatMapF { mime => + if (mime.matches(MimeType.text("plain"))) self.some.pure[F] + else alt(mime).as(None: Option[Path]) + }.value + + def readText: F[String] = + Files[F] + .readAll(self) + .through(fs2.text.utf8.decode) + .compile + .string + + def readAll: Stream[F, Byte] = + Files[F].readAll(self) + + def writeJson[A: Encoder](value: A): F[Unit] = + Stream + .emit(value.asJson.noSpaces) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(self)) + .compile + .drain + } +} + +object FileSupport extends FileSupport diff --git a/modules/files/src/main/scala/docspell/files/Zip.scala b/modules/files/src/main/scala/docspell/files/Zip.scala index 3fd938e4..b8d1dfd0 100644 --- a/modules/files/src/main/scala/docspell/files/Zip.scala +++ b/modules/files/src/main/scala/docspell/files/Zip.scala @@ -8,11 +8,12 @@ package docspell.files import java.io.InputStream import java.nio.charset.StandardCharsets -import java.nio.file.Paths import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} +import cats.data.OptionT import cats.effect._ import cats.implicits._ +import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common.Binary @@ -27,16 +28,72 @@ object Zip { ): Pipe[F, (String, Stream[F, Byte]), Byte] = in => zipJava(logger, chunkSize, in.through(deduplicate)) - def unzipP[F[_]: Async](chunkSize: Int, glob: Glob): Pipe[F, Byte, Binary[F]] = - s => unzip[F](chunkSize, glob)(s) + def unzip[F[_]: Async]( + chunkSize: Int, + glob: Glob + ): Pipe[F, Byte, Binary[F]] = + s => unzipStream[F](chunkSize, glob)(s) - def unzip[F[_]: Async](chunkSize: Int, glob: Glob)( + def unzipStream[F[_]: Async](chunkSize: Int, glob: Glob)( data: Stream[F, Byte] ): Stream[F, Binary[F]] = data .through(fs2.io.toInputStream[F]) .flatMap(in => unzipJava(in, chunkSize, glob)) + def saveTo[F[_]: Async]( + logger: Logger[F], + targetDir: Path, + moveUp: Boolean + ): Pipe[F, Binary[F], Path] = + binaries => + binaries + .filter(e => !e.name.endsWith("/")) + .evalMap { entry => + val out = targetDir / entry.name + val createParent = + OptionT + .fromOption[F](out.parent) + .flatMapF(parent => + Files[F] + .exists(parent) + .map(flag => Option.when(!flag)(parent)) + ) + .semiflatMap(p => Files[F].createDirectories(p)) + .getOrElse(()) + + logger.trace(s"Unzip ${entry.name} -> $out") *> + createParent *> + entry.data.through(Files[F].writeAll(out)).compile.drain + } + .drain ++ Stream + .eval(if (moveUp) moveContentsUp(logger)(targetDir) else ().pure[F]) + .as(targetDir) + + private def moveContentsUp[F[_]: Sync: Files](logger: Logger[F])(dir: Path): F[Unit] = + Files[F] + .list(dir) + .take(2) + .compile + .toList + .flatMap { + case subdir :: Nil => + Files[F].isDirectory(subdir).flatMap { + case false => ().pure[F] + case true => + Files[F] + .list(subdir) + .filter(p => p != dir) + .evalTap(c => logger.trace(s"Move $c -> ${dir / c.fileName}")) + .evalMap(child => Files[F].move(child, dir / child.fileName)) + .compile + .drain + } + + case _ => + ().pure[F] + } + def unzipJava[F[_]: Async]( in: InputStream, chunkSize: Int, @@ -55,7 +112,7 @@ object Zip { .unNoneTerminate .filter(ze => glob.matchFilenameOrPath(ze.getName())) .map { ze => - val name = Paths.get(ze.getName()).getFileName.toString + val name = ze.getName() val data = fs2.io.readInputStream[F]((zin: InputStream).pure[F], chunkSize, false) Binary(name, data) diff --git a/modules/files/src/test/resources/zip-dirs-one.zip b/modules/files/src/test/resources/zip-dirs-one.zip new file mode 100644 index 00000000..4a68b33a Binary files /dev/null and b/modules/files/src/test/resources/zip-dirs-one.zip differ diff --git a/modules/files/src/test/resources/zip-dirs.zip b/modules/files/src/test/resources/zip-dirs.zip new file mode 100644 index 00000000..816f8413 Binary files /dev/null and b/modules/files/src/test/resources/zip-dirs.zip differ diff --git a/modules/files/src/test/scala/docspell/files/ZipTest.scala b/modules/files/src/test/scala/docspell/files/ZipTest.scala index 6012e1c2..9230ac8f 100644 --- a/modules/files/src/test/scala/docspell/files/ZipTest.scala +++ b/modules/files/src/test/scala/docspell/files/ZipTest.scala @@ -7,20 +7,25 @@ package docspell.files import cats.effect._ -import cats.effect.unsafe.implicits.global import cats.implicits._ +import fs2.io.file.{Files, Path} import docspell.common.Glob +import docspell.logging.TestLoggingConfig import munit._ -class ZipTest extends FunSuite { +class ZipTest extends CatsEffectSuite with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + val tempDir = ResourceFixture( + Files[IO].tempDirectory(Path("target").some, "zip-test-", None) + ) test("unzip") { val zipFile = ExampleFiles.letters_zip.readURL[IO](8192) - val uncomp = zipFile.through(Zip.unzip(8192, Glob.all)) + val unzip = zipFile.through(Zip.unzip(8192, Glob.all)) - uncomp + unzip .evalMap { entry => val x = entry.data.map(_ => 1).foldMonoid.compile.lastOrError x.map { size => @@ -35,6 +40,10 @@ class ZipTest extends FunSuite { } .compile .drain - .unsafeRunSync() + } + + tempDir.test("unzipTo directory tree") { _ => + // val zipFile = ExampleFiles.zip_dirs_zip.readURL[IO](8192) + // zipFile.through(Zip.unzip(G)) } } diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 780d48b4..7a7d9b97 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -780,4 +780,75 @@ Docpell Update Check index-all-chunk = 10 } } + + addons { + # A directory to extract addons when running them. Everything in + # here will be cleared after each run. + working-dir = ${java.io.tmpdir}"/docspell-addons" + + # A directory for addons to store data between runs. This is not + # cleared by Docspell and can get large depending on the addons + # executed. + # + # This directory is used as base. In it subdirectories are created + # per run configuration id. + cache-dir = ${java.io.tmpdir}"/docspell-addon-cache" + + executor-config { + # Define a (comma or whitespace separated) list of runners that + # are responsible for executing an addon. This setting is + # compared to what is supported by addons. Possible values are: + # + # - nix-flake: use nix-flake runner if the addon supports it + # (this requires the nix package manager on the joex machine) + # - docker: use docker + # - trivial: use the trivial runner + # + # The first successful execution is used. This should list all + # runners the computer supports. + runner = "nix-flake, docker, trivial" + + # systemd-nspawn can be used to run the program in a container. + # This is used by runners nix-flake and trivial. + nspawn = { + # If this is false, systemd-nspawn is not tried. When true, the + # addon is executed inside a lightweight container via + # systemd-nspawn. + enabled = false + + # Path to sudo command. By default systemd-nspawn is executed + # via sudo - the user running joex must be allowed to do so NON + # INTERACTIVELY. If this is empty, then nspawn is tried to + # execute without sudo. + sudo-binary = "sudo" + + # Path to the systemd-nspawn command. + nspawn-binary = "systemd-nspawn" + + # Workaround, if multiple same named containers are run too fast + container-wait = "100 millis" + } + + # The timeout for running an addon. + run-timeout = "15 minutes" + + # Configure the nix flake runner. + nix-runner { + # Path to the nix command. + nix-binary = "nix" + + # The timeout for building the package (running nix build). + build-timeout = "15 minutes" + } + + # Configure the docker runner + docker-runner { + # Path to the docker command. + docker-binary = "docker" + + # The timeout for building the package (running docker build). + build-timeout = "15 minutes" + } + } + } } \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index de171135..646033c2 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.TextAnalysisConfig import docspell.analysis.classifier.TextClassifierConfig import docspell.backend.Config.Files +import docspell.backend.joex.AddonEnvConfig import docspell.common._ import docspell.config.{FtsType, PgFtsConfig} import docspell.convert.ConvertConfig @@ -43,7 +44,8 @@ case class Config( files: Files, mailDebug: Boolean, fullTextSearch: Config.FullTextSearch, - updateCheck: UpdateCheckConfig + updateCheck: UpdateCheckConfig, + addons: AddonEnvConfig ) { def pubSubConfig(headerValue: Ident): PubSubConfig = diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index c410cc42..e3a679a8 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -145,6 +145,8 @@ object JoexAppImpl extends MailAddressCodec { schedulerModule.scheduler, schedulerModule.periodicScheduler ) + nodes <- ONode(store) + _ <- nodes.withRegistered(cfg.appId, NodeType.Joex, cfg.baseUrl, None) appR <- Resource.make(app.init.map(_ => app))(_.initShutdown) } yield appR diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 5bdc8f18..ae1bf1a7 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -59,7 +59,7 @@ object JoexServer { Router("pubsub" -> pubSub.receiveRoute) }, "/api/info" -> InfoRoutes(cfg), - "/api/v1" -> JoexRoutes(joexApp) + "/api/v1" -> JoexRoutes(cfg, joexApp) ).orNotFound // With Middlewares in place diff --git a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala index 913ee930..f5c99d47 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala @@ -9,7 +9,9 @@ package docspell.joex import cats.effect.{Async, Resource} import docspell.analysis.TextAnalyser +import docspell.backend.BackendCommands import docspell.backend.fulltext.CreateIndex +import docspell.backend.joex.AddonOps import docspell.backend.ops._ import docspell.backend.task.DownloadZipArgs import docspell.common._ @@ -17,6 +19,7 @@ import docspell.config.FtsType import docspell.ftsclient.FtsClient import docspell.ftspsql.PsqlFtsClient import docspell.ftssolr.SolrFtsClient +import docspell.joex.addon.{ItemAddonTask, ScheduledAddonTask} import docspell.joex.analysis.RegexNerFile import docspell.joex.download.DownloadZipTask import docspell.joex.emptytrash.EmptyTrashTask @@ -32,6 +35,7 @@ import docspell.joex.preview.{AllPreviewsTask, MakePreviewTask} import docspell.joex.process.{ItemHandler, ReProcessItem} import docspell.joex.scanmailbox.ScanMailboxTask import docspell.joex.updatecheck.{ThisVersion, UpdateCheck, UpdateCheckTask} +import docspell.joexapi.client.JoexClient import docspell.notification.api.NotificationModule import docspell.pubsub.api.PubSubT import docspell.scheduler.impl.JobStoreModuleBuilder @@ -57,7 +61,8 @@ final class JoexTasks[F[_]: Async]( createIndex: CreateIndex[F], joex: OJoex[F], jobs: OJob[F], - itemSearch: OItemSearch[F] + itemSearch: OItemSearch[F], + addons: AddonOps[F] ) { val downloadAll: ODownloadAll[F] = ODownloadAll(store, jobs, jobStoreModule.jobs) @@ -68,7 +73,8 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, store, itemOps, fts, analyser, regexNer), + ItemHandler + .newItem[F](cfg, store, itemOps, fts, analyser, regexNer, addons), ItemHandler.onCancel[F](store) ) ) @@ -82,7 +88,15 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), + ReProcessItem[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addons, + store + ), ReProcessItem.onCancel[F] ) ) @@ -223,6 +237,20 @@ final class JoexTasks[F[_]: Async]( DownloadZipTask.onCancel[F] ) ) + .withTask( + JobTask.json( + ScheduledAddonTaskArgs.taskName, + ScheduledAddonTask[F](addons), + ScheduledAddonTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ItemAddonTaskArgs.taskName, + ItemAddonTask[F](addons, store), + ItemAddonTask.onCancel[F] + ) + ) } object JoexTasks { @@ -237,8 +265,9 @@ object JoexTasks { emailService: Emil[F] ): Resource[F, JoexTasks[F]] = for { - joex <- OJoex(pubSub) - store = jobStoreModule.store + store <- Resource.pure(jobStoreModule.store) + node <- ONode(store) + joex <- OJoex(pubSub, node, JoexClient(httpClient)) upload <- OUpload(store, jobStoreModule.jobs) fts <- createFtsClient(cfg, pools, store, httpClient) createIndex <- CreateIndex.resource(fts, store) @@ -250,6 +279,16 @@ object JoexTasks { notification <- ONotification(store, notificationModule) fileRepo <- OFileRepository(store, jobStoreModule.jobs) jobs <- OJob(store, joex, pubSub) + fields <- OCustomFields(store) + attachmentOps = OAttachment(store, fts, jobStoreModule.jobs) + cmdRunner = BackendCommands(itemOps, attachmentOps, fields, notification, None) + addons = AddonOps( + cfg.addons, + store, + cmdRunner, + attachmentOps, + jobStoreModule.jobs + ) } yield new JoexTasks[F]( cfg, store, @@ -266,7 +305,8 @@ object JoexTasks { createIndex, joex, jobs, - itemSearchOps + itemSearchOps, + addons ) private def createFtsClient[F[_]: Async]( diff --git a/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala new file mode 100644 index 00000000..0b1bad57 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.MonadError + +import docspell.addons.AddonExecutionResult +import docspell.scheduler.PermanentError + +trait AddonTaskExtension { + implicit final class AddonExecutionResultOps(self: AddonExecutionResult) { + def raiseErrorIfNeeded[F[_]](implicit m: MonadError[F, Throwable]): F[Unit] = + if (self.isFailure && self.pure) { + m.raiseError(new Exception(s"Addon execution failed: $self")) + } else if (self.isFailure) { + m.raiseError( + PermanentError( + new Exception( + "Addon execution failed. Do not retry, because some addons were impure." + ) + ) + ) + } else m.pure(()) + + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala new file mode 100644 index 00000000..50bbf41b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Files + +import docspell.addons.{AddonTriggerType, InputEnv, Middleware} +import docspell.backend.joex.AddonOps.ExecResult +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.common._ +import docspell.files.FileSupport +import docspell.joex.process.ItemData +import docspell.logging.Logger +import docspell.scheduler.Task +import docspell.store.Store +import docspell.store.queries.QAttachment + +object GenericItemAddonTask extends LoggerExtension with FileSupport { + + private val itemSubdir = "item" + private val itemDataJson = s"$itemSubdir/item-data.json" + private val argsMetaJson = s"$itemSubdir/given-data.json" + private val pdfDir = s"$itemSubdir/pdfs" + private val originalDir = s"$itemSubdir/originals" + private val originalMetaJson = s"$itemSubdir/source-files.json" + private val pdfMetaJson = s"$itemSubdir/pdf-files.json" + + // This environment can be used by the addon to access data of the current task + private val itemEnv = Map( + "ITEM_DIR" -> itemSubdir, + "ITEM_DATA_JSON" -> itemDataJson, + "ITEM_ARGS_JSON" -> argsMetaJson, + "ITEM_PDF_DIR" -> pdfDir, + "ITEM_ORIGINAL_DIR" -> originalDir, + "ITEM_ORIGINAL_JSON" -> originalMetaJson, + "ITEM_PDF_JSON" -> pdfMetaJson + ) + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ItemData] = + addonResult(ops, store, trigger, addonTaskIds)(collective, data, maybeMeta).as( + data + ) + + def addonResult[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ExecResult] = + Task { ctx => + ops.execAll(collective, Set(trigger), addonTaskIds, ctx.logger.some)( + Middleware.prepare(Kleisli(prepareItemData(ctx.logger, store, data, maybeMeta))) + ) + } + + def prepareItemData[F[_]: Async]( + logger: Logger[F], + store: Store[F], + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + )( + input: InputEnv + ): F[InputEnv] = + for { + _ <- logger.debug(s"Preparing item data '${data.item.name}' for addon") + wd = input.baseDir + itemMetaFile = wd / itemDataJson + argsMetaFile = wd / argsMetaJson + pdfs = wd / pdfDir + originals = wd / originalDir + srcJson = wd / originalMetaJson + pdfJson = wd / pdfMetaJson + + _ <- List(wd / itemSubdir, pdfs, originals).traverse(Files[F].createDirectories) + + _ <- logger.debug("Writing collected item data…") + _ <- itemMetaFile.writeJson(data) + + _ <- OptionT + .fromOption[F](maybeMeta) + .semiflatMap { meta => + logger.debug("Writing context meta data…") *> + argsMetaFile.writeJson(meta) + } + .value + + _ <- logger.debug("Storing all attachments…") + _ <- data.attachments + .flatMap(a => + Vector( + pdfs / a.id.id -> a.fileId, + originals / a.id.id -> data.originFile(a.id) + ) + ) + .traverse_ { case (out, key) => + logger.debug(s"Storing attachment $out") *> + store.fileRepo + .getBytes(key) + .through(Files[F].writeAll(out)) + .compile + .drain + } + + _ <- logger.debug("Storing file metadata") + srcMeta <- store.transact(QAttachment.attachmentSourceFile(data.item.id)) + pdfMeta <- store.transact(QAttachment.attachmentFile(data.item.id)) + _ <- srcJson.writeJson(srcMeta) + _ <- pdfJson.writeJson(pdfMeta) + } yield input.addEnv(itemEnv) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala new file mode 100644 index 00000000..32f8bc7b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.{ItemAddonTaskArgs, MetaProposalList} +import docspell.joex.process.ItemData +import docspell.scheduler.{PermanentError, Task} +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +object ItemAddonTask extends AddonTaskExtension { + type Args = ItemAddonTaskArgs + val name = ItemAddonTaskArgs.taskName + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + + def apply[F[_]: Async](ops: AddonOps[F], store: Store[F]): Task[F, Args, Result] = + Task { ctx => + (for { + item <- OptionT( + store.transact( + RItem.findByIdAndCollective(ctx.args.itemId, ctx.args.collective) + ) + ) + data <- OptionT.liftF(makeItemData(store, item)) + inner = GenericItemAddonTask.addonResult( + ops, + store, + AddonTriggerType.ExistingItem, + ctx.args.addonRunConfigs + )(ctx.args.collective, data, None) + execResult <- OptionT.liftF(inner.run(ctx.unit)) + _ <- OptionT.liftF(execResult.combined.raiseErrorIfNeeded[F]) + } yield Result( + execResult.combined.addonResult, + execResult.runConfigs.flatMap(_.refs).map(_.archive.nameAndVersion).distinct + )).getOrElseF( + Async[F].raiseError( + PermanentError( + new NoSuchElementException(s"Item not found for id: ${ctx.args.itemId.id}!") + ) + ) + ) + } + + def makeItemData[F[_]: Async](store: Store[F], item: RItem): F[ItemData] = + for { + attachs <- store.transact(RAttachment.findByItem(item.id)) + rmeta <- store.transact(QAttachment.getAttachmentMetaOfItem(item.id)) + rsource <- store.transact(RAttachmentSource.findByItem(item.id)) + proposals <- store.transact(QAttachment.getMetaProposals(item.id, item.cid)) + tags <- store.transact(RTag.findByItem(item.id)) + } yield ItemData( + item = item, + attachments = attachs, + metas = rmeta, + dateLabels = Vector.empty, + originFile = rsource.map(r => (r.id, r.fileId)).toMap, + givenMeta = proposals, + tags = tags.map(_.name).toList, + classifyProposals = MetaProposalList.empty, + classifyTags = Nil + ) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/Result.scala b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala new file mode 100644 index 00000000..fdd86ba2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import docspell.addons.AddonResult +import docspell.scheduler.JobTaskResultEncoder + +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder + +case class Result(addonResult: AddonResult, addons: List[String]) + +object Result { + val empty: Result = + Result(AddonResult.empty, Nil) + + implicit val jsonEncoder: Encoder[Result] = + deriveEncoder + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[Result] = + JobTaskResultEncoder.fromJson[Result].withMessage { result => + result.addonResult match { + case AddonResult.Success(_) => + s"Executed ${result.addons.size} addon(s) successfully." + + case AddonResult.ExecutionError(rc) => + s"Addon execution finished with non-zero return code: $rc" + + case AddonResult.ExecutionFailed(ex) => + s"Addon execution failed: ${ex.getMessage}" + + case AddonResult.DecodingError(msg) => + s"Addon output failed to read: $msg" + } + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala new file mode 100644 index 00000000..7f108223 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.backend.ops.OAddons +import docspell.common.{Ident, ScheduledAddonTaskArgs} +import docspell.scheduler.Task + +object ScheduledAddonTask extends AddonTaskExtension with LoggerExtension { + type Args = ScheduledAddonTaskArgs + + val name: Ident = OAddons.scheduledAddonTaskName + + def apply[F[_]: Async](ops: AddonOps[F]): Task[F, Args, Result] = + Task { ctx => + for { + execRes <- ops.execById(ctx.args.collective, ctx.args.addonTaskId, ctx.logger)( + Middleware.identity[F] + ) + _ <- execRes.result.combineAll.raiseErrorIfNeeded[F] + } yield Result( + execRes.result.combineAll.addonResult, + execRes.runConfigs.flatMap(_.refs.map(_.archive.nameAndVersion)) + ) + } + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala index 4e6d9f52..ea0500c6 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.split.TextSplitter import docspell.common._ +import docspell.common.util.File import docspell.store.queries.QCollective import io.circe.generic.semiauto._ diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 2f46234e..8d3f3562 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.store.Store import docspell.store.queries.QCollective import docspell.store.records.REquipment diff --git a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala index 1d61f4fd..24244d6b 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala @@ -14,6 +14,7 @@ import fs2.io.file.Path import docspell.analysis.classifier.{ClassifierModel, TextClassifier} import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger import docspell.store.Store import docspell.store.records.RClassifierModel diff --git a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala index 6fc5f634..0470fcdc 100644 --- a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala @@ -109,7 +109,7 @@ object MultiUploadArchiveTask { )(file: ProcessItemArgs.File): Stream[F, ProcessItemArgs] = store.fileRepo .getBytes(file.fileMetaId) - .through(Zip.unzipP[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) + .through(Zip.unzip[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) .flatMap { entry => val hint = MimeTypeHint(entry.name.some, entry.mime.asString.some) entry.data diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index 31c1e007..98a2923b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -69,7 +69,7 @@ object AttachmentPreview { } case mt => - ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot get page count.") *> + ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot create preview.") *> (None: Option[RAttachmentPreview]).pure[F] } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index 5e2d86b0..27cbe414 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -146,7 +146,7 @@ object ExtractArchive { val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all) ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *> zipData - .through(Zip.unzipP[F](8192, glob)) + .through(Zip.unzip[F](8192, glob)) .zipWithIndex .flatMap(handleEntry(ctx, store, ra, pos, archive, None)) .foldMonoid diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index c96c0189..94a6c07f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -26,7 +26,8 @@ import io.circe.{Encoder, Json} * @param dateLabels * a separate list of found dates * @param originFile - * a mapping from an attachment id to a filemeta-id containng the source or origin file + * a mapping from an attachment id to a filemeta-id containing the source or origin + * file * @param givenMeta * meta data to this item that was not "guessed" from an attachment but given and thus * is always correct @@ -49,7 +50,7 @@ case class ItemData( ) { /** sort by weight; order of equal weights is not important, just choose one others are - * then suggestions doc-date is only set when given explicitely, not from "guessing" + * then suggestions doc-date is only set when given explicitly, not from "guessing" */ def finalProposals: MetaProposalList = MetaProposalList @@ -98,7 +99,7 @@ object ItemData { dates.map(dl => dl.label.copy(label = dl.date.toString)) } - // Used to encode the result passed to the job-done event + // Used to encode the result passed to the job-done event and to supply to addons implicit val jsonEncoder: Encoder[ItemData] = Encoder.instance { data => val metaMap = data.metas.groupMap(_.id)(identity) @@ -108,10 +109,12 @@ object ItemData { "collective" -> data.item.cid.asJson, "source" -> data.item.source.asJson, "attachments" -> data.attachments + .sortBy(_.position) .map(a => Json.obj( "id" -> a.id.asJson, "name" -> a.name.asJson, + "position" -> a.position.asJson, "content" -> metaMap.get(a.id).flatMap(_.head.content).asJson, "language" -> metaMap.get(a.id).flatMap(_.head.language).asJson, "pages" -> metaMap.get(a.id).flatMap(_.head.pages).asJson @@ -123,6 +126,18 @@ object ItemData { "assumedCorrOrg" -> data.finalProposals .find(MetaProposalType.CorrOrg) .map(_.values.head.ref) + .asJson, + "assumedCorrPerson" -> data.finalProposals + .find(MetaProposalType.CorrPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcPerson" -> data.finalProposals + .find(MetaProposalType.ConcPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcEquip" -> data.finalProposals + .find(MetaProposalType.ConcEquip) + .map(_.values.head.ref) .asJson ) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 8d59a969..b5fc216e 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.Stream import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} import docspell.ftsclient.FtsClient @@ -41,7 +42,8 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] ): Task[F, Args, Option[ItemData]] = logBeginning[F].flatMap(_ => DuplicateCheck[F](store) @@ -52,7 +54,17 @@ object ItemHandler { CreateItem[F](store).contramap(_ => args.pure[F]) create .flatMap(itemStateTask(store, ItemState.Processing)) - .flatMap(safeProcess[F](cfg, store, itemOps, fts, analyser, regexNer)) + .flatMap( + safeProcess[F]( + cfg, + store, + itemOps, + fts, + analyser, + regexNer, + addons + ) + ) .map(_.some) } ) @@ -76,11 +88,14 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data).attempt + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -91,7 +106,9 @@ object ItemHandler { .andThen(_ => Sync[F].raiseError(ex)) } case false => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data) + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ) .flatMap(itemStateTask(store, ItemState.Created)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 6087b37f..836a8062 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -9,7 +9,9 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs import docspell.ftsclient.FtsClient @@ -26,6 +28,7 @@ object ProcessItem { fts: FtsClient[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = ExtractArchive(store)(item) @@ -35,6 +38,7 @@ object ProcessItem { .flatMap(SetGivenData.onlyNew[F](itemOps)) .flatMap(Task.setProgress(99)) .flatMap(RemoveEmptyItem(itemOps)) + .flatMap(RunAddons(addonOps, store, AddonTriggerType.FinalProcessItem)) def processAttachments[F[_]: Async]( cfg: Config, diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 1863d2ef..6890e37d 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -10,7 +10,9 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common._ import docspell.ftsclient.FtsClient @@ -34,13 +36,24 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] ): Task[F, Args, Unit] = Task .log[F, Args](_.info("===== Start reprocessing ======")) .flatMap(_ => loadItem[F](store) - .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer, store)) + .flatMap( + safeProcess[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store + ) + ) .map(_ => ()) ) @@ -99,6 +112,7 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F], data: ItemData ): Task[F, Args, ItemData] = { @@ -129,6 +143,7 @@ object ReProcessItem { .processAttachments[F](cfg, fts, analyser, regexNer, store)(data) .flatMap(LinkProposal[F](store)) .flatMap(SetGivenData[F](itemOps)) + .flatMap(RunAddons[F](addonOps, store, AddonTriggerType.FinalReprocessItem)) .contramap[Args](convertArgs(lang)) } } @@ -153,11 +168,21 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data).attempt + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -167,7 +192,16 @@ object ReProcessItem { ).andThen(_ => Sync[F].raiseError(ex)) } case false => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data) + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ) } private def logWarn[F[_]](msg: => String): Task[F, Args, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala new file mode 100644 index 00000000..b564d8b4 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.process + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.ProcessItemArgs +import docspell.joex.addon.GenericItemAddonTask +import docspell.scheduler.Task +import docspell.store.Store + +/** Run registered addons in the context of item processing. The addon has access to the + * current item data and can apply custom processing logic. + */ +object RunAddons { + type Args = ProcessItemArgs + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType + )( + data: ItemData + ): Task[F, Args, ItemData] = + if (data.item.state.isInvalid && data.attachments.isEmpty) { + Task.pure(data) + } else + Task { ctx => + val inner = GenericItemAddonTask(ops, store, trigger, Set.empty)( + ctx.args.meta.collective, + data, + ctx.args.meta.some + ) + inner.run(ctx.unit) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala index 0734d294..7fd35abe 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -44,7 +44,7 @@ object SetGivenData { for { _ <- ctx.logger.info("Starting setting given data") _ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'") - e <- ops.setFolder(itemId, folderId, collective).attempt + e <- ops.setFolder(itemId, folderId.map(_.id), collective).attempt _ <- e.fold( ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"), res => diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala index 6810f3d9..e02ff31d 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Duration, Ident, Timestamp} -import docspell.joex.JoexApp +import docspell.joex.{Config, JoexApp} import docspell.joexapi.model._ import docspell.store.records.RJobLog @@ -20,7 +20,7 @@ import org.http4s.dsl.Http4sDsl object JoexRoutes { - def apply[F[_]: Async](app: JoexApp[F]): HttpRoutes[F] = { + def apply[F[_]: Async](cfg: Config, app: JoexApp[F]): HttpRoutes[F] = { val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { @@ -64,6 +64,11 @@ object JoexRoutes { BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found") ) } yield resp + + case GET -> Root / "addon" / "config" => + val data = + AddonSupport(cfg.appId, cfg.addons.executorConfig.runner) + Ok(data) } } diff --git a/modules/joexapi/src/main/resources/joex-openapi.yml b/modules/joexapi/src/main/resources/joex-openapi.yml index 27bb62eb..c0e1f55a 100644 --- a/modules/joexapi/src/main/resources/joex-openapi.yml +++ b/modules/joexapi/src/main/resources/joex-openapi.yml @@ -122,8 +122,45 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /addon/config: + get: + operationId: "v1-addon-config-get" + tags: [ Addons ] + summary: What is supported running addons + description: | + Return what this joex supports when executing addons. + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonSupport" + components: schemas: + AddonSupport: + description: | + How this joex supports executing addons. + required: + - nodeId + - runners + properties: + nodeId: + type: string + format: ident + runners: + type: array + items: + type: string + format: addon-runner-type + enum: + - nix-flake + - docker + - trivial + JobAndLog: description: | Some more details about the job. diff --git a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala index c623eba0..b9b5b5ef 100644 --- a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala +++ b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Ident, LenientUri} -import docspell.joexapi.model.BasicResult +import docspell.joexapi.model.{AddonSupport, BasicResult} import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.circe.CirceEntityDecoder @@ -25,6 +25,7 @@ trait JoexClient[F[_]] { def cancelJob(base: LenientUri, job: Ident): F[BasicResult] + def getAddonSupport(base: LenientUri): F[AddonSupport] } object JoexClient { @@ -33,6 +34,13 @@ object JoexClient { new JoexClient[F] with CirceEntityDecoder { private[this] val logger = docspell.logging.getLogger[F] + def getAddonSupport(base: LenientUri): F[AddonSupport] = { + val getUrl = base / "api" / "v1" / "addon" / "config" + val req = Request[F](Method.GET, uri(getUrl)) + logger.debug(s"Getting addon support") *> + client.expect[AddonSupport](req) + } + def notifyJoex(base: LenientUri): F[BasicResult] = { val notifyUrl = base / "api" / "v1" / "notify" val req = Request[F](Method.POST, uri(notifyUrl)) diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala index 83f320bb..f5aaee15 100644 --- a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala +++ b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala @@ -143,7 +143,9 @@ object Event { } - /** Some generic list of items, chosen by a user. */ + /** Some generic list of items, chosen by a user. This is use to notify about periodic + * search results. + */ final case class ItemSelection( account: AccountId, items: Nel[Ident], diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 7a509842..ab9cc8a9 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -5790,9 +5790,388 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /sec/addon/archive: + get: + operationId: "sec-addon-archive-get" + tags: [Addons] + summary: Get all registered addons + description: | + Returns a list of all registered addons. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonList" + post: + operationId: "sec-addon-post" + tags: [ Addons ] + summary: Install a new addon + description: | + Given an URL to an addon (which is a zip file containing a + `docspell-meta.yaml` or json descriptor), the addon is + downloaded and installed in docspell. + + By default this happens asynchronously and the response only + indicates that installing has been submitted. The result will + be transfered over the websocket channel. With query parameter + `sync` installing happens synchronously and it may take a + while to complete (if successful, the addon id is returned). + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRegister" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + /sec/addon/archive/{id}: + parameters: + - $ref: "#/components/parameters/id" + delete: + operationId: "sec-addon-archive-delete" + tags: [Addons] + summary: Deletes the addon and removes it from all addon run configs + description: | + Deletes the addon from the database and also removes it from + all run configurations where it might be referenced. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + operationId: "sec-addon-archive-put" + tags: [ Addons ] + summary: Update an addon from its url + description: | + Addons are urls to zip files. This call reads the url again + and updates the contents in docspell for this addon. + + By default this happens asynchronously and the response only + indicates that updating has been submitted. The result will be + transfered over the websocket channel. With query parameter + `sync` updating happens synchronously and it may take a while + to complete. + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run-config: + get: + operationId: "sec-addon-run-config-get" + tags: [Addons] + summary: Get all addon run configs + description: | + Returns a list of addon run configs. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfigList" + post: + operationId: "sec-addon-run-config-post" + tags: [ Addons ] + summary: Adds a new addon run config + description: | + Adds a new set of configured addons, creating a run + configuration. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + + /sec/addon/run-config/{id}: + parameters: + - $ref: "#/components/parameters/id" + put: + operationId: "sec-addon-run-config-id-put" + tags: [ Addons ] + summary: Updates an addon run config + description: | + Updates an existing addon run configuration. The id is taken + from the URL and any given id in the request body is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + delete: + operationId: "sec-addonrunconfig-delete" + tags: [Addons] + summary: Deletes the addon run config given its id + description: | + Deletes the addon run configuration. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run/existingitem: + post: + operationId: "sec-addon-run-existing-item" + tags: [Addons] + summary: Submits a task running addons for an item + description: | + Submits a background task that executes the specified (or all) + addons configured to use for an existing item. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunExistingItem" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" components: schemas: + AddonRunExistingItem: + description: | + Data to run addons for an existing item. + required: + - itemId + properties: + itemId: + type: string + format: ident + additionalItems: + type: array + items: + type: string + format: ident + description: | + Additional items to run addons on. There will be one job + submitted per item. + addonRunConfigIds: + type: array + items: + type: string + format: ident + description: | + If non empty, only select these addon run configs. + Otherwise all configured to be run for existing items are + executed. + + AddonRef: + description: | + A reference to an addon (archive) with additional name and + version and its arguments. When used for adding addon run + configs, name and version are ignored. + required: + - addonId + - name + - version + - args + properties: + addonId: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + args: + type: string + + AddonRunConfig: + description: | + A set of configured addons that are run on certain points + defined by the `trigger` property. + required: + - id + - name + - enabled + - trigger + - addons + properties: + id: + type: string + format: ident + name: + type: string + enabled: + type: boolean + userId: + type: string + format: ident + description: | + An addon can be run on behalf of a user. If not given, no + authentication token is generated into the environment of + the addon. The user can be given as user_id or by its + login name. + schedule: + type: string + format: calevent + description: | + A schedule must be supplied when a trigger type of + 'scheduled' is defined. + trigger: + description: | + Defines when this task is executed. There must be at least + one element. Possible values: + + * process-item: After an item has been processed + * reprocess-item: After an item has been re-processed + * scheduled: Executed periodically based on a schedule, + which must be defined then + type: array + items: + type: string + format: addon-trigger-type + enum: + - process-item + - reprocess-item + - scheduled + addons: + type: array + items: + $ref: "#/components/schemas/AddonRef" + + AddonRunConfigList: + description: | + A list of addon run configurations. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/AddonRunConfig" + + AddonRegister: + description: | + Data to register addons + required: + - url + properties: + url: + type: string + format: uri + + Addon: + description: | + An registered addon. + required: + - id + - name + - version + - created + properties: + id: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + url: + type: string + format: uri + created: + type: integer + format: date-time + + AddonList: + description: | + A list of addons + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Addon" + DownloadAllSummary: description: | Information about a ZIP download. diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 508bb277..2c6d4523 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -462,5 +462,39 @@ docspell.server { } } } + + addons = { + enabled = false + + # Whether installing addons requiring network should be allowed + # or not. + allow-impure = true + + # Define patterns of urls that are allowed to install addons + # from. + # + # A pattern is compared against an URL by comparing three parts + # of an URL via globs: scheme, host and path. + # + # You can use '*' (0 or more) and '?' (one) as wildcards in each + # part. For example: + # + # https://*.mydomain.com/projects/* + # *s://gitea.mydomain/* + # + # A hostname is separated by dots and the path by a slash. A '*' + # in a pattern means to match one or more characters. The path + # pattern is always matching the given prefix. So /a/b/* matches + # /a/b/c and /a/b/c/d and all other sub-paths. + # + # Multiple patterns can be defined va a comma separated string + # or as an array. An empty string matches no URL, while the + # special pattern '*' all by itself means to match every URL. + allowed-urls = "*" + + # Same as `allowed-urls` but a match here means do deny addons + # from this url. + denied-urls = "" + } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 5d2b26a6..45f58c2e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -152,7 +152,9 @@ final class RestAppImpl[F[_]: Async]( "clientSettings" -> ClientSettingsRoutes(backend, token), "notification" -> NotificationRoutes(config, backend, token), "querybookmark" -> BookmarkRoutes(backend, token), - "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token) + "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token), + "addonrunconfig" -> AddonRunConfigRoutes(backend, token), + "addon" -> AddonRoutes(config, wsTopic, backend, token) ) } @@ -181,7 +183,16 @@ object RestAppImpl { .withEventSink(notificationMod) .build backend <- BackendApp - .create[F](store, javaEmil, ftsClient, pubSubT, schedulerMod, notificationMod) + .create[F]( + cfg.backend, + store, + javaEmil, + httpClient, + ftsClient, + pubSubT, + schedulerMod, + notificationMod + ) app = new RestAppImpl[F]( cfg, diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 1a8fd6c4..e4c1a5bc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -14,6 +14,7 @@ import fs2.Stream import fs2.concurrent.Topic import docspell.backend.msg.Topics +import docspell.backend.ops.ONode import docspell.common._ import docspell.pubsub.naive.NaivePubSub import docspell.restserver.http4s.InternalHeader @@ -91,6 +92,15 @@ object RestServer { store, httpClient )(Topics.all.map(_.topic)) + + nodes <- ONode(store) + _ <- nodes.withRegistered( + cfg.appId, + NodeType.Restserver, + cfg.baseUrl, + cfg.auth.serverSecret.some + ) + restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic) } yield (restApp, pubSub, setting) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala new file mode 100644 index 00000000..dd2351b8 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.conv + +import cats.syntax.all._ + +import docspell.addons.AddonMeta +import docspell.backend.ops.AddonValidationError +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.Ident +import docspell.restserver.ws.{OutputEvent, OutputEventEncoder} +import docspell.store.records.RAddonArchive + +trait AddonValidationSupport { + + def validationErrorToMessage(e: AddonValidationError): String = + e match { + case AddonValidationError.AddonNotFound => + "Addon not found." + + case AddonValidationError.AddonExists(msg, _) => + msg + + case AddonValidationError.NotAnAddon(ex) => + s"The url doesn't seem to be an addon: ${ex.getMessage}" + + case AddonValidationError.InvalidAddon(msg) => + s"The addon is not valid: $msg" + + case AddonValidationError.AddonUnsupported(msg, _) => + msg + + case AddonValidationError.AddonsDisabled => + "Addons are disabled in the config file." + + case AddonValidationError.UrlUntrusted(_) => + "This url doesn't belong to te set of trusted urls defined in the config file" + + case AddonValidationError.DownloadFailed(ex) => + s"Downloading the addon failed: ${ex.getMessage}" + + case AddonValidationError.ImpureAddonsDisabled => + s"Installing impure addons is disabled." + + case AddonValidationError.RefreshLocalAddon => + "Refreshing a local addon doesn't work." + } + + def addonResultOutputEventEncoder( + collective: Ident + ): OutputEventEncoder[AddonValidationResult[(RAddonArchive, AddonMeta)]] = + OutputEventEncoder.instance { + case Right((archive, _)) => + OutputEvent.AddonInstalled( + collective, + "Addon installed", + None, + archive.id.some, + archive.originalUrl + ) + + case Left(error) => + val msg = validationErrorToMessage(error) + OutputEvent.AddonInstalled(collective, msg, error.some, None, None) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala index aead3504..6b17d49e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala @@ -47,4 +47,8 @@ object Responses { def notFoundRoute[F[_]: Sync]: HttpRoutes[F] = HttpRoutes(_ => OptionT.pure(Response.notFound[F])) + def notFoundRoute[F[_]: Sync, A](body: A)(implicit + entityEncoder: EntityEncoder[F, A] + ): HttpRoutes[F] = + HttpRoutes(_ => OptionT.pure(Response.notFound[F].withEntity(body))) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala new file mode 100644 index 00000000..ceeabadc --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.http4s + +import cats.effect._ + +import docspell.joexapi.model.BasicResult + +import org.http4s.Response +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +trait ThrowableResponseMapper { + + implicit class EitherThrowableOps[A](self: Either[Throwable, A]) { + def rightAs[F[_]: Sync](f: A => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], f) + + def rightAs_[F[_]: Sync](r: => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], _ => r) + } +} + +object ThrowableResponseMapper { + def toResponse[F[_]: Sync](ex: Throwable): F[Response[F]] = + new Mapper[F].toResponse(ex) + + private class Mapper[F[_]: Sync] extends Http4sDsl[F] { + def toResponse(ex: Throwable): F[Response[F]] = + ex match { + case _: IllegalArgumentException => + BadRequest(BasicResult(false, ex.getMessage)) + + case _ => + InternalServerError(BasicResult(false, ex.getMessage)) + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala new file mode 100644 index 00000000..72c9b808 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.addons.AddonMeta +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.conv.AddonValidationSupport +import docspell.restserver.ws.{Background, OutputEvent} +import docspell.store.records.RAddonArchive + +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl +import org.http4s.dsl.impl.FlagQueryParamMatcher +import org.http4s.{HttpRoutes, Response} + +object AddonArchiveRoutes extends AddonValidationSupport { + + def apply[F[_]: Async]( + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + implicit val wsOutputEnc = addonResultOutputEventEncoder(token.account.collective) + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddons(token.account.collective) + resp <- Ok( + AddonList( + all.map(r => + Addon(r.id, r.name, r.version, r.description, r.originalUrl, r.created) + ) + ) + ) + } yield resp + + case req @ POST -> Root :? Sync(sync) => + def create(r: Option[RAddonArchive]) = + IdResult( + true, + r.fold("Addon submitted for installation")(r => + s"Addon installed: ${r.id.id}" + ), + r.map(_.id).getOrElse(Ident.unsafe("")) + ) + + for { + input <- req.as[AddonRegister] + install = backend.addons.registerAddon( + token.account.collective, + input.url, + None + ) + resp <- + if (sync) + install.flatMap( + _.fold(convertAddonValidationError[F], r => Ok(create(r._1.some))) + ) + else Background(wsTopic)(install).flatMap(_ => Ok(create(None))) + } yield resp + + case PUT -> Root / Ident(id) :? Sync(sync) => + def create(r: Option[AddonMeta]) = + BasicResult( + true, + r.fold("Addon updated in background")(m => + s"Addon updated: ${m.nameAndVersion}" + ) + ) + val update = backend.addons.refreshAddon(token.account.collective, id) + for { + resp <- + if (sync) + update.flatMap( + _.fold( + convertAddonValidationError[F], + r => Ok(create(r._2.some)) + ) + ) + else Background(wsTopic)(update).flatMap(_ => Ok(create(None))) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddon(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon deleted")) + else NotFound(BasicResult(false, "Addon not found")) + } yield resp + } + } + + def convertAddonValidationError[F[_]: Async]( + e: AddonValidationError + ): F[Response[F]] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + + def failWith(msg: String): F[Response[F]] = + Ok(IdResult(false, msg, Ident.unsafe(""))) + + e match { + case AddonValidationError.AddonNotFound => + NotFound(BasicResult(false, "Addon not found.")) + + case _ => + failWith(validationErrorToMessage(e)) + } + } + + object Sync extends FlagQueryParamMatcher("sync") +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala new file mode 100644 index 00000000..192f6ccb --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect.Async +import fs2.concurrent.Topic + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model.BasicResult +import docspell.restserver.Config +import docspell.restserver.http4s.Responses +import docspell.restserver.ws.OutputEvent + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.server.Router + +object AddonRoutes { + + def apply[F[_]: Async]( + cfg: Config, + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = + if (cfg.backend.addons.enabled) + Router( + "archive" -> AddonArchiveRoutes(wsTopic, backend, token), + "run-config" -> AddonRunConfigRoutes(backend, token), + "run" -> AddonRunRoutes(backend, token) + ) + else + Responses.notFoundRoute(BasicResult(false, "Addons disabled")) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala new file mode 100644 index 00000000..d146eef1 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.OAddons +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunConfigRoutes { + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddonRunConfigs(token.account.collective) + resp <- Ok(AddonRunConfigList(all.map(convertInfoTask))) + } yield resp + + case req @ POST -> Root => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(Ident.unsafe(""), input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config added", id)) + ) + } yield resp + + case req @ PUT -> Root / Ident(id) => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(id, input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config updated", id)) + ) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddonRunConfig(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon task deleted")) + else NotFound(BasicResult(false, "Addon task not found")) + } yield resp + } + } + + def convertInsertTask( + id: Ident, + t: AddonRunConfig + ): Either[String, OAddons.AddonRunInsert] = + for { + tr <- NonEmptyList + .fromList(t.trigger) + .toRight("At least one trigger is required") + ta <- NonEmptyList + .fromList(t.addons) + .toRight("At least one addon is required") + res = OAddons.AddonRunInsert( + id, + t.name, + t.enabled, + t.userId, + t.schedule, + tr, + ta.map(e => OAddons.AddonArgs(e.addonId, e.args)) + ) + } yield res + + def convertInfoTask(t: OAddons.AddonRunInfo): AddonRunConfig = + AddonRunConfig( + id = t.id, + name = t.name, + enabled = t.enabled, + userId = t.userId, + schedule = t.schedule, + trigger = t.triggered, + addons = t.addons.map { case (ra, raa) => + AddonRef(raa.addonId, ra.name, ra.version, ra.description, raa.args) + } + ) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala new file mode 100644 index 00000000..3b167b2d --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunRoutes { + + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { case req @ POST -> Root / "existingitem" => + for { + input <- req.as[AddonRunExistingItem] + _ <- backend.addons.runAddonForItem( + token.account, + NonEmptyList(input.itemId, input.additionalItems), + input.addonRunConfigIds.toSet + ) + resp <- Ok(BasicResult(true, "Job for running addons submitted.")) + } yield resp + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala index a713f160..1a8ebd6a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala @@ -117,7 +117,11 @@ object ItemMultiRoutes extends NonEmptyListSupport with MultiIdSupport { for { json <- req.as[ItemsAndRef] items <- requireNonEmpty(json.items) - res <- backend.item.setFolderMultiple(items, json.ref, user.account.collective) + res <- backend.item.setFolderMultiple( + items, + json.ref.map(_.id), + user.account.collective + ) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index d830fc6c..bcabb4c7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -218,7 +218,7 @@ object ItemRoutes { case req @ PUT -> Root / Ident(id) / "folder" => for { idref <- req.as[OptionalId] - res <- backend.item.setFolder(id, idref.id, user.account.collective) + res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala index de54198c..5faf9f76 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala @@ -29,7 +29,8 @@ case class Flags( downloadAllMaxFiles: Int, downloadAllMaxSize: ByteSize, uiVersion: Int, - openIdAuth: List[Flags.OpenIdAuth] + openIdAuth: List[Flags.OpenIdAuth], + addonsEnabled: Boolean ) object Flags { @@ -47,7 +48,8 @@ object Flags { cfg.downloadAll.maxFiles, cfg.downloadAll.maxSize, uiVersion, - cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)) + cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)), + cfg.backend.addons.enabled ) final case class OpenIdAuth(provider: Ident, name: String) diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala new file mode 100644 index 00000000..b0870fc5 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.logging.Logger + +/** Asynchronous operations that run on the rest-server can communicate their results via + * websocket. + */ +object Background { + // TODO avoid resubmitting same stuff + + def apply[F[_]: Async, A]( + wsTopic: Topic[F, OutputEvent], + logger: Option[Logger[F]] = None + )(run: F[A])(implicit enc: OutputEventEncoder[A]): F[Unit] = { + val log = logger.getOrElse(docspell.logging.getLogger[F]) + Async[F] + .background(run) + .use( + _.flatMap( + _.fold( + log.warn("The background operation has been cancelled!"), + ex => log.error(ex)("Error running background operation!"), + event => + event + .map(enc.encode) + .flatTap(ev => log.info(s"Sending response from async operation: $ev")) + .flatMap(wsTopic.publish1) + .void + ) + ) + ) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala index c7e57be4..5ac68c48 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala @@ -7,6 +7,7 @@ package docspell.restserver.ws import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError import docspell.common._ import io.circe._ @@ -55,6 +56,29 @@ object OutputEvent { Msg("jobs-waiting", count).asJson } + final case class AddonInstalled( + collective: Ident, + message: String, + error: Option[AddonValidationError], + addonId: Option[Ident], + originalUrl: Option[LenientUri] + ) extends OutputEvent { + def forCollective(token: AuthToken) = + token.account.collective == collective + + override def asJson = + Msg( + "addon-installed", + Map( + "success" -> error.isEmpty.asJson, + "error" -> error.asJson, + "addonId" -> addonId.asJson, + "addonUrl" -> originalUrl.asJson, + "message" -> message.asJson + ) + ).asJson + } + private case class Msg[A](tag: String, content: A) private object Msg { implicit def jsonEncoder[A: Encoder]: Encoder[Msg[A]] = diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala new file mode 100644 index 00000000..4414dfad --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +trait OutputEventEncoder[A] { + def encode(a: A): OutputEvent +} + +object OutputEventEncoder { + def apply[A](implicit e: OutputEventEncoder[A]): OutputEventEncoder[A] = e + + def instance[A](f: A => OutputEvent): OutputEventEncoder[A] = + (a: A) => f(a) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala index 0594dcf8..4ca51dc9 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -6,6 +6,8 @@ package docspell.scheduler +import cats.effect.Sync + import docspell.common._ import docspell.logging.Logger @@ -25,4 +27,8 @@ trait Context[F[_], A] { self => def map[C](f: A => C): Context[F, C] + def unit: Context[F, Unit] = + map(_ => ()) + + def loadJob(implicit F: Sync[F]): F[Job[String]] } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala new file mode 100644 index 00000000..7a921769 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +/** Special "marker" exception to indicate errors in tasks, that should NOT be retried. */ +final class PermanentError(cause: Throwable) extends RuntimeException(cause) { + override def fillInStackTrace() = this +} + +object PermanentError { + def apply(cause: Throwable): PermanentError = + new PermanentError(cause) + + def isPermanent(ex: Throwable): Boolean = + unapply(ex).isDefined + + def unapply(ex: Throwable): Option[Throwable] = + ex match { + case p: PermanentError => Some(p.getCause) + case _ => None + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala index d6868a08..b8ecd5fb 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala @@ -21,6 +21,11 @@ trait Task[F[_], A, B] { def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = Task(Task.toKleisli(this).andThen(f)) + def andThenC[C](f: (Context[F, A], B) => F[C])(implicit M: Monad[F]): Task[F, A, C] = { + val run = Task.toKleisli(this).run + Task(ctx => run(ctx).flatMap(b => f(ctx, b))) + } + def mapF[C](f: F[B] => F[C]): Task[F, A, C] = Task(Task.toKleisli(this).mapF(f)) diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala index 236c7ee6..bd3027fe 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala @@ -50,6 +50,9 @@ object UserTaskScope { def apply(collective: Ident): UserTaskScope = UserTaskScope.collective(collective) + def apply(collective: Ident, login: Option[Ident]): UserTaskScope = + login.map(AccountId(collective, _)).map(account).getOrElse(apply(collective)) + def system: UserTaskScope = collective(DocspellSystem.taskGroup) } diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala index 59016b9f..7b5a7615 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -24,6 +24,19 @@ class ContextImpl[F[_]: Functor, A]( val jobId: Ident ) extends Context[F, A] { + def loadJob(implicit F: Sync[F]): F[Job[String]] = + JobStoreImpl(store) + .findById(jobId) + .flatMap( + _.fold( + F.raiseError[Job[String]]( + new IllegalStateException(s"Job not found: ${jobId.id}") + ) + )( + F.pure + ) + ) + def setProgress(percent: Int): F[Unit] = { val pval = math.min(100, math.max(0, percent)) store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala index 5ebfd665..be263ad1 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala @@ -34,6 +34,7 @@ object LogSink { .capture("task", e.taskName) .capture("group", e.group) .capture("jobInfo", e.jobInfo) + .captureAll(e.data) e.level match { case LogLevel.Info => diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala index 50410e65..7e81cd1d 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala @@ -12,8 +12,7 @@ import cats.syntax.all._ import fs2.Stream import docspell.common.{Ident, LogLevel} -import docspell.logging -import docspell.logging.{Level, Logger} +import docspell.logging.{Level, LogEvent => DsLogEvent, Logger} /** Background tasks use this logger to emit the log events to a queue. The consumer is * [[LogSink]], which picks up log events in a separate thread. @@ -29,7 +28,7 @@ object QueueLogger { ): Logger[F] = new Logger[F] { - def log(logEvent: => logging.LogEvent) = + def log(logEvent: => DsLogEvent) = LogEvent .create[F]( jobId, diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index bc87d0fa..c114e2bc 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -267,27 +267,36 @@ final class SchedulerImpl[F[_]: Async]( .mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa) .mapF(_.attempt.flatMap { case Right(result) => - logger.info(s"Job execution successful: ${job.info}") - ctx.logger.info("Job execution successful") *> + logger.info(s"Job execution successful: ${job.info}") *> + ctx.logger.info("Job execution successful") *> (JobState.Success: JobState, result).pure[F] + + case Left(PermanentError(ex)) => + logger.warn(ex)("Task failed with permanent error") *> + ctx.logger + .warn(ex)("Task failed with permanent error!") + .as(JobState.failed -> JobTaskResult.empty) + case Left(ex) => state.get.map(_.wasCancelled(job)).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") - ctx.logger.error(ex)("Job execution failed (cancel = true)") *> + logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") *> + ctx.logger.error(ex)("Job execution failed (cancel = true)") *> (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] case false => QJob.exceedsRetries(job.id, config.retries, store).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) case false => - logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retrying later.") - .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") + .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) } } }) diff --git a/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql new file mode 100644 index 00000000..4bc97aba --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table `addon_archive`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `file_id` varchar(254) not null, + `original_url` varchar(2000), + `name` varchar(254) not null, + `version` varchar(254) not null, + `description` text, + `triggers` text not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`file_id`) references `filemeta`(`file_id`), + unique (`cid`, `original_url`), + unique (`cid`, `name`, `version`) +); + +create table `addon_run_config`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `user_id` varchar(254), + `name` varchar(254) not null, + `enabled` boolean not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`user_id`) references `user_`(`uid`) +); + +create table `addon_run_config_addon` ( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `addon_id` varchar(254) not null, + `args` text not null, + `position` int not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + foreign key (`addon_id`) references `addon_archive`(`id`) on delete cascade +); + +create table `addon_run_config_trigger`( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `triggers` varchar(254) not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + unique (`addon_run_config_id`, `triggers`) +); + +alter table `node` +add column (`server_secret` varchar(2000)); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala new file mode 100644 index 00000000..df69f421 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.file + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream + +import docspell.common.{FileKey, LenientUri, UrlReader} + +import binny.BinaryId + +object FileUrlReader { + + private val scheme: String = "docspell-file" + + def url(key: FileKey): LenientUri = + LenientUri( + scheme = Nel.of(scheme), + authority = Some(""), + path = LenientUri.NonEmptyPath( + Nel.of(key.collective.id, key.category.id.id, key.id.id) + ), + query = None, + fragment = None + ) + + def apply[F[_]: Sync](repo: FileRepository[F]): UrlReader[F] = + UrlReader.instance { url => + url.scheme.head match { + case `scheme` => + Stream + .emit(urlToFileKey(url)) + .covary[F] + .rethrow + .evalMap(key => repo.findMeta(key).map(m => (key, m))) + .flatMap { + case _ -> Some(m) => repo.getBytes(m.id) + case key -> None => + Stream.raiseError( + new NoSuchElementException( + s"File not found for url '${url.asString}' (key=$key)" + ) + ) + } + + case _ => + UrlReader.defaultReader[F].apply(url) + } + } + + private[file] def urlToFileKey(url: LenientUri): Either[Throwable, FileKey] = + BinnyUtils + .binaryIdToFileKey(BinaryId(url.host match { + case Some(h) if h.nonEmpty => s"$h${url.path.asString}" + case _ => url.path.segments.mkString("/") + })) + .leftMap(new IllegalArgumentException(_)) +} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala index 738a078b..6e919330 100644 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala @@ -9,6 +9,7 @@ package docspell.store.impl import java.time.format.DateTimeFormatter import java.time.{Instant, LocalDate} +import docspell.addons.AddonTriggerType import docspell.common._ import docspell.common.syntax.all._ import docspell.jsonminiq.JsonMiniQuery @@ -31,9 +32,9 @@ trait DoobieMeta extends EmilDoobieMeta { implicit val sqlLogging: LogHandler = LogHandler { case e @ Success(_, _, _, _) => - DoobieMeta.logger.trace("SQL " + e) + DoobieMeta.logger.trace(s"SQL: $e") case e => - DoobieMeta.logger.error(s"SQL Failure: $e") + DoobieMeta.logger.warn(s"SQL Failure: $e") } def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = @@ -41,6 +42,12 @@ trait DoobieMeta extends EmilDoobieMeta { e.apply(a).noSpaces ) + implicit val metaAddonTriggerType: Meta[AddonTriggerType] = + Meta[String].timap(AddonTriggerType.unsafeFromString)(_.name) + + implicit val metaAddonTriggerTypeSet: Meta[Set[AddonTriggerType]] = + jsonMeta[Set[AddonTriggerType]] + implicit val metaBinaryId: Meta[BinaryId] = Meta[String].timap(BinaryId.apply)(_.id) diff --git a/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala new file mode 100644 index 00000000..94e51f96 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.queries + +import docspell.common.BaseJsonCodecs._ +import docspell.common._ + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} +import scodec.bits.ByteVector + +/** Information about an attachment file (can be attachment-source or attachment) */ +case class AttachedFile( + id: Ident, + name: Option[String], + position: Int, + language: Option[Language], + mimetype: MimeType, + length: ByteSize, + checksum: ByteVector +) + +object AttachedFile { + + implicit val jsonDecoder: Decoder[AttachedFile] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachedFile] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 5cdcafb3..6e90cc86 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -22,10 +22,37 @@ import doobie._ object QAttachment { private val a = RAttachment.as("a") + private val as = RAttachmentSource.as("ats") private val item = RItem.as("i") private val am = RAttachmentMeta.as("am") private val c = RCollective.as("c") private val im = RItemProposal.as("im") + private val fm = RFileMeta.as("fm") + + def attachmentSourceFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(as.id, as.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(as, a.id === as.id) + .innerJoin(fm, fm.id === as.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] + + def attachmentFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(a.id, a.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(fm, fm.id === a.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] def deletePreview[F[_]: Sync](store: Store[F])(attachId: Ident): F[Int] = { val findPreview = @@ -163,6 +190,17 @@ object QAttachment { q.query[RAttachmentMeta].option } + def getAttachmentMetaOfItem(itemId: Ident): ConnectionIO[Vector[RAttachmentMeta]] = + Select( + select(am.all), + from(am) + .innerJoin(a, a.id === am.id), + a.itemId === itemId + ).orderBy(a.position.asc) + .build + .query[RAttachmentMeta] + .to[Vector] + case class ContentAndName( id: Ident, item: Ident, @@ -175,6 +213,7 @@ object QAttachment { def allAttachmentMetaAndName( coll: Option[Ident], itemIds: Option[Nel[Ident]], + itemStates: Nel[ItemState], chunkSize: Int ): Stream[ConnectionIO, ContentAndName] = Select( @@ -192,7 +231,7 @@ object QAttachment { .innerJoin(item, item.id === a.itemId) .innerJoin(c, c.id === item.cid) ).where( - item.state.in(ItemState.validStates) &&? + item.state.in(itemStates) &&? itemIds.map(ids => item.id.in(ids)) &&? coll.map(cid => item.cid === cid) ).build diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala new file mode 100644 index 00000000..69c6826a --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala @@ -0,0 +1,155 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.AddonTriggerType +import docspell.common.{Ident, Timestamp} +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ + +case class AddonRunConfigData( + runConfig: RAddonRunConfig, + addons: List[RAddonRunConfigAddon], + triggers: List[RAddonRunConfigTrigger] +) + +object AddonRunConfigData { + + def findAll( + cid: Ident, + enabled: Option[Boolean] = None, + trigger: Set[AddonTriggerType] = Set.empty, + configIds: Set[Ident] = Set.empty + ): ConnectionIO[List[AddonRunConfigData]] = + for { + runConfigs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + addons <- runConfigs.traverse(t => + RAddonRunConfigAddon.findByRunConfig(t.id).map(as => t.id -> as) + ) + addonMap = addons.toMap + triggers <- runConfigs.traverse(t => + RAddonRunConfigTrigger.findByRunConfig(t.id).map(ts => t.id -> ts) + ) + triggerMap = triggers.toMap + result = runConfigs.map(t => + AddonRunConfigData(t, addonMap(t.id), triggerMap(t.id)) + ) + } yield result + + /** Inserts new, creating new identifiers */ + def insert(task: AddonRunConfigData): ConnectionIO[Ident] = + for { + tid <- Ident.randomId[ConnectionIO] + now <- Timestamp.current[ConnectionIO] + tr = task.runConfig.copy(id = tid, created = now) + _ <- RAddonRunConfig.insert(tr) + _ <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigTrigger.insert) + } + _ <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield tid + + /** Updates the task, keeping its id but replacing all related objects */ + def update(task: AddonRunConfigData): ConnectionIO[Int] = + for { + n1 <- RAddonRunConfig.update(task.runConfig) + _ <- RAddonRunConfigTrigger.deleteAllForConfig(task.runConfig.id) + _ <- RAddonRunConfigAddon.deleteAllForConfig(task.runConfig.id) + tts <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigTrigger.insert) + } + tas <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield n1 + tts.sum + tas.sum + + def findEnabledRef( + cid: Ident, + taskId: Ident + ): ConnectionIO[List[(RAddonArchive, RAddonRunConfigAddon)]] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + + Select( + combineNel(select(aa.all), select(ta.all)), + from(run) + .innerJoin(ta, ta.runConfigId === run.id) + .innerJoin(aa, aa.id === ta.addonId), + run.cid === cid && run.enabled === true && run.id === taskId + ).orderBy(ta.position.asc) + .build + .query[(RAddonArchive, RAddonRunConfigAddon)] + .to[List] + } + + def findEnabledRefs( + cid: Ident, + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + ): Stream[ConnectionIO, (RAddonRunConfig, List[(RAddonArchive, String)])] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + val tt = RAddonRunConfigTrigger.as("tt") + + val taskIdFilter = NonEmptyList + .fromList(addonTaskIds.toList) + .map(nel => run.id.in(nel)) + val validTasks = TableDef("valid_task") + val validTaskId = Column[Ident]("id", validTasks) + val query = + withCte( + validTasks -> Select( + select(run.all), + from(run) + .innerJoin(tt, tt.runConfigId === run.id), + run.cid === cid && run.enabled === true && tt.trigger === trigger &&? taskIdFilter + ).distinct + )( + Select( + combineNel( + select(run.all.map(_.copy(table = validTasks))), + select(aa.all), + select(ta.args) + ), + from(validTasks) + .innerJoin(ta, ta.runConfigId === validTaskId) + .innerJoin(aa, aa.id === ta.addonId) + ).orderBy(validTaskId) + ).build + + query + .query[(RAddonRunConfig, RAddonArchive, String)] + .stream + .groupAdjacentBy(_._1.id) + .map { case (_, chunk) => + val list = chunk.toList + (list.head._1, list.map(e => (e._2, e._3))) + } + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala new file mode 100644 index 00000000..94ddb6bc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.OptionT +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ +import docspell.store.records.AddonRunConfigResolved.AddonRef + +import doobie._ +import doobie.implicits._ + +final case class AddonRunConfigResolved( + config: RAddonRunConfig, + refs: List[AddonRef], + trigger: List[RAddonRunConfigTrigger] +) {} + +object AddonRunConfigResolved { + + case class AddonRef(archive: RAddonArchive, ref: RAddonRunConfigAddon) + + def findAddonRefs(configId: Ident): ConnectionIO[List[AddonRef]] = { + val ca = RAddonRunConfigAddon.as("ca") + val aa = RAddonArchive.as("aa") + Select( + select(combineNel(aa.all, ca.all)), + from(ca) + .innerJoin(aa, aa.id === ca.addonId), + ca.runConfigId === configId + ).build.query[AddonRef].to[List] + } + + def getRefsAndTrigger( + configId: Ident + ): ConnectionIO[(List[AddonRef], List[RAddonRunConfigTrigger])] = + (findAddonRefs(configId), RAddonRunConfigTrigger.findByRunConfig(configId)).tupled + + def findById( + configId: Ident, + collective: Ident, + enabled: Option[Boolean] + ): ConnectionIO[Option[AddonRunConfigResolved]] = + (for { + cfg <- OptionT(RAddonRunConfig.findById(collective, configId)) + .filter(c => enabled.isEmpty || enabled == c.enabled.some) + (refs, tri) <- OptionT.liftF(getRefsAndTrigger(configId)) + } yield AddonRunConfigResolved(cfg, refs, tri)).value + + def findAllForCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[AddonRunConfigResolved]] = + for { + cfgs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + result <- cfgs.traverse(ac => + getRefsAndTrigger(ac.id).map { case (refs, tri) => + AddonRunConfigResolved(ac, refs, tri) + } + ) + } yield result +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala new file mode 100644 index 00000000..05e2fc15 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} +import docspell.common._ +import docspell.store.file.FileUrlReader +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class RAddonArchive( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + name: String, + version: String, + description: Option[String], + triggers: Set[AddonTriggerType], + created: Timestamp +) { + + def nameAndVersion: String = + s"${name}-${version}" + + def isUnchanged(meta: AddonMeta): Boolean = + name == meta.meta.name && + version == meta.meta.version && + description == meta.meta.description + + def isChanged(meta: AddonMeta): Boolean = + !isUnchanged(meta) + + def asArchive: AddonArchive = + AddonArchive(FileUrlReader.url(fileId), name, version) + + def update(file: FileKey, meta: AddonMeta): RAddonArchive = + copy( + fileId = file, + name = meta.meta.name, + version = meta.meta.version, + description = meta.meta.description, + triggers = meta.triggers.getOrElse(Set.empty) + ) +} + +object RAddonArchive { + case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_archive" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val fileId = Column[FileKey]("file_id", this) + val originalUrl = Column[LenientUri]("original_url", this) + val name = Column[String]("name", this) + val version = Column[String]("version", this) + val description = Column[String]("description", this) + val triggers = Column[Set[AddonTriggerType]]("triggers", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of( + id, + cid, + fileId, + originalUrl, + name, + version, + description, + triggers, + created + ) + } + + def apply( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + meta: AddonMeta, + created: Timestamp + ): RAddonArchive = + RAddonArchive( + id, + cid, + fileId, + originalUrl, + meta.meta.name, + meta.meta.version, + meta.meta.description, + meta.triggers.getOrElse(Set.empty), + created + ) + + def as(alias: String): Table = + Table(Some(alias)) + + val T = Table(None) + + def insert(r: RAddonArchive, silent: Boolean): ConnectionIO[Int] = { + val values = + sql"${r.id}, ${r.cid}, ${r.fileId}, ${r.originalUrl}, ${r.name}, ${r.version}, ${r.description}, ${r.triggers}, ${r.created}" + + if (silent) DML.insertSilent(T, T.all, values) + else DML.insert(T, T.all, values) + } + + def existsByUrl(cid: Ident, url: LenientUri): ConnectionIO[Boolean] = + Select( + select(count(T.id)), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[Int].unique.map(_ > 0) + + def findByUrl(cid: Ident, url: LenientUri): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[RAddonArchive].option + + def findByNameAndVersion( + cid: Ident, + name: String, + version: String + ): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.name === name && T.version === version + ).build.query[RAddonArchive].option + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id === id + ).build.query[RAddonArchive].option + + def findByIds(cid: Ident, ids: NonEmptyList[Ident]): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id.in(ids) + ).orderBy(T.name).build.query[RAddonArchive].to[List] + + def update(r: RAddonArchive): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id && T.cid === r.cid, + DML.set( + T.fileId.setTo(r.fileId), + T.originalUrl.setTo(r.originalUrl), + T.name.setTo(r.name), + T.version.setTo(r.version), + T.description.setTo(r.description), + T.triggers.setTo(r.triggers) + ) + ) + + def listAll(cid: Ident): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid + ).orderBy(T.name.asc).build.query[RAddonArchive].to[List] + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) + + implicit val jsonDecoder: Decoder[RAddonArchive] = deriveDecoder + implicit val jsonEncoder: Encoder[RAddonArchive] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala new file mode 100644 index 00000000..70460aaa --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfig( + id: Ident, + cid: Ident, + userId: Option[Ident], + name: String, + enabled: Boolean, + created: Timestamp +) + +object RAddonRunConfig { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val userId = Column[Ident]("user_id", this) + val name = Column[String]("name", this) + val enabled = Column[Boolean]("enabled", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, cid, userId, name, enabled, created) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfig): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.cid}, ${r.userId}, ${r.name}, ${r.enabled}, ${r.created}" + ) + + def update(r: RAddonRunConfig): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.name.setTo(r.name), + T.enabled.setTo(r.enabled), + T.userId.setTo(r.userId) + ) + ) + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonRunConfig]] = + Select(select(T.all), from(T), T.cid === cid && T.id === id).build + .query[RAddonRunConfig] + .option + + def findByCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[RAddonRunConfig]] = { + val ac = RAddonRunConfig.as("ac") + val tt = RAddonRunConfigTrigger.as("tt") + val filter = + ac.cid === cid &&? + enabled.map(e => ac.enabled === e) &&? + NonEmptyList.fromList(configIds.toList).map(ids => ac.id.in(ids)) + + val selectConfigs = + NonEmptyList.fromList(trigger.toList) match { + case Some(tri) => + Select( + select(ac.all), + from(ac).innerJoin(tt, tt.runConfigId === ac.id), + filter && tt.trigger.in(tri) + ) + case None => + Select(select(ac.all), from(ac), filter) + } + + selectConfigs.build.query[RAddonRunConfig].to[List] + } + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala new file mode 100644 index 00000000..aa95ca35 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigAddon( + id: Ident, + runConfigId: Ident, + addonId: Ident, + args: String, + position: Int +) + +object RAddonRunConfigAddon { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_addon" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val addonId = Column[Ident]("addon_id", this) + val args = Column[String]("args", this) + val position = Column[Int]("position", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, addonId, args, position) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfigAddon): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.runConfigId}, ${r.addonId}, ${r.args}, ${r.position}" + ) + + def updateArgs(addonTaskId: Ident, addonId: Ident, args: String): ConnectionIO[Int] = + DML.update( + T, + T.runConfigId === addonTaskId && T.addonId === addonId, + DML.set( + T.args.setTo(args) + ) + ) + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigAddon]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId) + .orderBy(T.position.asc) + .build + .query[RAddonRunConfigAddon] + .to[List] + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala new file mode 100644 index 00000000..fccd81f2 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigTrigger( + id: Ident, + runConfigId: Ident, + trigger: AddonTriggerType +) + +object RAddonRunConfigTrigger { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_trigger" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val trigger = Column[AddonTriggerType]("triggers", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, trigger) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) + + def insert(r: RAddonRunConfigTrigger): ConnectionIO[Int] = + DML.insert(T, T.all, sql"${r.id}, ${r.runConfigId}, ${r.trigger}") + + def insertAll( + addonTaskId: Ident, + triggers: NonEmptyList[AddonTriggerType] + ): ConnectionIO[Int] = { + val records = triggers.traverse(t => + Ident.randomId[ConnectionIO].map(id => RAddonRunConfigTrigger(id, addonTaskId, t)) + ) + val inserts = + s"INSERT INTO ${T.tableName} (id, addon_run_config_id, trigger) VALUES (?,?,?)" + records.flatMap(rs => Update[RAddonRunConfigTrigger](inserts).updateMany(rs)) + } + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigTrigger]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId).build + .query[RAddonRunConfigTrigger] + .to[List] +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 64599d6e..cb53f3d8 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -119,6 +119,9 @@ object RAttachmentMeta { def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] = DML.update(T, T.id === mid, DML.set(T.pages.setTo(pageCount))) + def updateContent(id: Ident, text: String): ConnectionIO[Int] = + DML.update(T, T.id === id, DML.set(T.content.setTo(text))) + def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index 3b9d23aa..6ca4bc8e 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -6,7 +6,8 @@ package docspell.store.records -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, OptionT} +import cats.syntax.all._ import docspell.common.{FileKey, _} import docspell.store.qb.DSL._ @@ -44,12 +45,29 @@ object RAttachmentPreview { def insert(v: RAttachmentPreview): ConnectionIO[Int] = DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") + def update(r: RAttachmentPreview): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.fileId.setTo(r.fileId), + T.name.setTo(r.name) + ) + ) + def findById(attachId: Ident): ConnectionIO[Option[RAttachmentPreview]] = run(select(T.all), from(T), T.id === attachId).query[RAttachmentPreview].option def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) + def upsert(r: RAttachmentPreview): ConnectionIO[Option[FileKey]] = + OptionT(findById(r.id)) + .semiflatMap(existing => + update(existing.copy(fileId = r.fileId, name = r.name)).as(Some(existing.fileId)) + ) + .getOrElseF(insert(r).as(None)) + def findByIdAndCollective( attachId: Ident, collective: Ident diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 17faebba..2ac753ca 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -309,20 +309,22 @@ object RItem { def updateFolder( itemId: Ident, coll: Ident, - folderId: Option[Ident] - ): ConnectionIO[Int] = + folderIdOrName: Option[String] + ): ConnectionIO[(Int, Option[Ident])] = for { t <- currentTime - fid <- folderId match { - case Some(f) => RFolder.requireIdByIdOrName(f, f.id, coll).map(_.some) - case None => None.pure[ConnectionIO] + fid <- folderIdOrName match { + case Some(f) => + val fid = Ident.fromString(f).getOrElse(Ident.unsafe("")) + RFolder.requireIdByIdOrName(fid, f, coll).map(_.some) + case None => None.pure[ConnectionIO] } n <- DML.update( T, T.cid === coll && T.id === itemId, DML.set(T.folder.setTo(fid), T.updated.setTo(t)) ) - } yield n + } yield (n, fid) def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = for { @@ -334,6 +336,26 @@ object RItem { ) } yield n + def appendNotes( + itemId: Ident, + cid: Ident, + text: String, + sep: Option[String] + ): ConnectionIO[Option[String]] = { + val curNotes = + Select(select(T.notes), from(T), T.cid === cid && T.id === itemId).build + .query[Option[String]] + .option + + curNotes.flatMap { + case Some(notes) => + val newText = notes.map(_ + sep.getOrElse("")).getOrElse("") + text + updateNotes(itemId, cid, Some(newText)).as(newText.some) + case None => + (None: Option[String]).pure[ConnectionIO] + } + } + def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = for { t <- currentTime diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index fdcb8dd4..ce3c1daa 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -16,6 +16,7 @@ import docspell.store.qb._ import doobie._ import doobie.implicits._ +import scodec.bits.ByteVector case class RNode( id: Ident, @@ -23,13 +24,19 @@ case class RNode( url: LenientUri, updated: Timestamp, created: Timestamp, - notFound: Int + notFound: Int, + serverSecret: Option[ByteVector] ) {} object RNode { - def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] = - Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0)) + def apply[F[_]: Sync]( + id: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[RNode] = + Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0, serverSecret)) final case class Table(alias: Option[String]) extends TableDef { val tableName = "node" @@ -40,7 +47,9 @@ object RNode { val updated = Column[Timestamp]("updated", this) val created = Column[Timestamp]("created", this) val notFound = Column[Int]("not_found", this) - val all = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created, notFound) + val serverSecret = Column[ByteVector]("server_secret", this) + val all = NonEmptyList + .of[Column[_]](id, nodeType, url, updated, created, notFound, serverSecret) } def as(alias: String): Table = @@ -52,7 +61,7 @@ object RNode { DML.insert( t, t.all, - fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound}" + fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound},${v.serverSecret}" ) } @@ -65,6 +74,7 @@ object RNode { DML.set( t.nodeType.setTo(v.nodeType), t.url.setTo(v.url), + t.serverSecret.setTo(v.serverSecret), t.updated.setTo(v.updated) ) ) diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index c100f19d..651c99fe 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -152,6 +152,14 @@ object RUser { .query[Ident] .option + case class IdAndLogin(uid: Ident, login: Ident) + def getIdByIdOrLogin(idOrLogin: Ident): ConnectionIO[Option[IdAndLogin]] = + Select( + select(T.uid, T.login), + from(T), + T.uid === idOrLogin || T.login === idOrLogin + ).build.query[IdAndLogin].option + def getIdByAccount(account: AccountId): ConnectionIO[Ident] = OptionT(findIdByAccount(account)).getOrElseF( Sync[ConnectionIO].raiseError( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cd832f82..372e708e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -264,6 +264,9 @@ object Dependencies { val circeGenericExtra = Seq( "io.circe" %% "circe-generic-extras" % CirceVersion ) + val circeYaml = Seq( + "io.circe" %% "circe-yaml" % CirceVersion + ) // // https://github.com/Log4s/log4s;ASL 2.0 // val loggingApi = Seq(