From 7fdd78ad06c44180b4ffdb42d2acd5cac82aae36 Mon Sep 17 00:00:00 2001 From: eikek Date: Fri, 22 Apr 2022 14:07:28 +0200 Subject: [PATCH] Experiment with addons Addons allow to execute external programs in some context inside docspell. Currently it is possible to run them after processing files. Addons are provided by URLs to zip files. --- build.sbt | 44 +- .../scala/docspell/addons/AddonArchive.scala | 90 ++++ .../addons/AddonExecutionResult.scala | 33 ++ .../scala/docspell/addons/AddonExecutor.scala | 121 +++++ .../docspell/addons/AddonExecutorConfig.scala | 45 ++ .../addons/AddonLoggerExtension.scala | 28 ++ .../scala/docspell/addons/AddonMeta.scala | 216 +++++++++ .../main/scala/docspell/addons/AddonRef.scala | 9 + .../scala/docspell/addons/AddonResult.scala | 114 +++++ .../scala/docspell/addons/AddonRunner.scala | 110 +++++ .../docspell/addons/AddonTriggerType.scala | 55 +++ .../main/scala/docspell/addons/Context.scala | 72 +++ .../scala/docspell/addons/Directory.scala | 74 +++ .../main/scala/docspell/addons/InputEnv.scala | 32 ++ .../scala/docspell/addons/Middleware.scala | 43 ++ .../scala/docspell/addons/RunnerType.scala | 69 +++ .../docspell/addons/out/AddonOutput.scala | 44 ++ .../scala/docspell/addons/out/ItemFile.scala | 102 +++++ .../scala/docspell/addons/out/NewFile.scala | 77 ++++ .../scala/docspell/addons/out/NewItem.scala | 92 ++++ .../main/scala/docspell/addons/package.scala | 15 + .../docspell/addons/runner/CollectOut.scala | 43 ++ .../addons/runner/DockerBuilder.scala | 82 ++++ .../docspell/addons/runner/DockerRunner.scala | 93 ++++ .../addons/runner/NSpawnBuilder.scala | 83 ++++ .../addons/runner/NixFlakeRunner.scala | 123 +++++ .../docspell/addons/runner/RunnerUtil.scala | 171 +++++++ .../addons/runner/TrivialRunner.scala | 78 ++++ .../resources/docspell-dummy-addon-master.zip | Bin 0 -> 7634 bytes .../docspell/addons/AddonArchiveTest.scala | 44 ++ .../docspell/addons/AddonExecutorTest.scala | 63 +++ .../scala/docspell/addons/AddonMetaTest.scala | 37 ++ .../docspell/addons/AddonOutputTest.scala | 40 ++ .../docspell/addons/AddonRunnerTest.scala | 98 ++++ .../test/scala/docspell/addons/Fixtures.scala | 71 +++ .../classifier/StanfordTextClassifier.scala | 2 +- .../docspell/analysis/nlp/PipelineCache.scala | 1 + .../StanfordTextClassifierSuite.scala | 1 + .../nlp/StanfordNerAnnotatorSuite.scala | 1 + .../docspell/backend/AttachedEvent.scala | 1 + .../scala/docspell/backend/BackendApp.scala | 30 +- .../docspell/backend/BackendCommands.scala | 175 +++++++ .../main/scala/docspell/backend/Config.scala | 20 +- .../scala/docspell/backend/JobFactory.scala | 20 + .../backend/fulltext/CreateIndex.scala | 9 +- .../backend/joex/AddonEnvConfig.scala | 17 + .../docspell/backend/joex/AddonOps.scala | 199 ++++++++ .../backend/joex/AddonPostProcess.scala | 198 ++++++++ .../docspell/backend/joex/AddonPrepare.scala | 75 +++ .../backend/joex/LoggerExtension.scala | 18 + .../backend/ops/AddonRunConfigError.scala | 48 ++ .../backend/ops/AddonRunConfigValidate.scala | 54 +++ .../docspell/backend/ops/AddonValidate.scala | 156 +++++++ .../backend/ops/AddonValidationError.scala | 85 ++++ .../scala/docspell/backend/ops/OAddons.scala | 426 ++++++++++++++++++ .../docspell/backend/ops/OAttachment.scala | 223 +++++++++ .../scala/docspell/backend/ops/OItem.scala | 90 +++- .../scala/docspell/backend/ops/OJoex.scala | 28 +- .../scala/docspell/backend/ops/ONode.scala | 40 +- .../docspell/common/BaseJsonCodecs.scala | 10 +- .../main/scala/docspell/common/Binary.scala | 12 + .../scala/docspell/common/FileCategory.scala | 4 +- .../src/main/scala/docspell/common/Glob.scala | 21 +- .../main/scala/docspell/common/Ident.scala | 3 + .../docspell/common/ItemAddonTaskArgs.scala | 28 ++ .../scala/docspell/common/MimeTypeHint.scala | 5 + .../docspell/common/ProcessItemArgs.scala | 2 +- .../common/ScheduledAddonTaskArgs.scala | 19 + .../scala/docspell/common/SystemCommand.scala | 69 ++- .../scala/docspell/common/UrlMatcher.scala | 94 ++++ .../scala/docspell/common/UrlReader.scala | 35 ++ .../docspell/common/bc/AttachmentAction.scala | 30 ++ .../docspell/common/bc/BackendCommand.scala | 44 ++ .../common/bc/BackendCommandRunner.scala | 17 + .../scala/docspell/common/bc/ItemAction.scala | 102 +++++ .../scala/docspell/common/exec/Args.scala | 49 ++ .../main/scala/docspell/common/exec/Env.scala | 37 ++ .../scala/docspell/common/exec/SysCmd.scala | 43 ++ .../scala/docspell/common/exec/SysExec.scala | 163 +++++++ .../docspell/common/{ => util}/File.scala | 13 +- .../scala/docspell/common/util/Random.scala | 27 ++ .../test/scala/docspell/common/GlobTest.scala | 2 + .../docspell/common/UrlMatcherTest.scala | 60 +++ .../common/bc/BackendCommandTest.scala | 85 ++++ .../scala/docspell/config/Implicits.scala | 12 + .../docspell/convert/extern/ExternConv.scala | 1 + .../docspell/convert/ConversionTest.scala | 1 + .../scala/docspell/convert/FileChecks.scala | 1 + .../convert/extern/ExternConvTest.scala | 1 + .../main/scala/docspell/extract/ocr/Ocr.scala | 1 + .../docspell/extract/ocr/OcrConfig.scala | 1 + .../scala/docspell/files/FileSupport.scala | 61 +++ .../src/main/scala/docspell/files/Zip.scala | 67 ++- .../files/src/test/resources/zip-dirs-one.zip | Bin 0 -> 1306 bytes modules/files/src/test/resources/zip-dirs.zip | Bin 0 -> 1098 bytes .../test/scala/docspell/files/ZipTest.scala | 19 +- .../joex/src/main/resources/reference.conf | 71 +++ .../src/main/scala/docspell/joex/Config.scala | 4 +- .../scala/docspell/joex/JoexAppImpl.scala | 2 + .../main/scala/docspell/joex/JoexServer.scala | 2 +- .../main/scala/docspell/joex/JoexTasks.scala | 52 ++- .../joex/addon/AddonTaskExtension.scala | 30 ++ .../joex/addon/GenericItemAddonTask.scala | 130 ++++++ .../docspell/joex/addon/ItemAddonTask.scala | 76 ++++ .../scala/docspell/joex/addon/Result.scala | 40 ++ .../joex/addon/ScheduledAddonTask.scala | 39 ++ .../docspell/joex/analysis/NerFile.scala | 1 + .../docspell/joex/analysis/RegexNerFile.scala | 1 + .../scala/docspell/joex/learn/Classify.scala | 1 + .../multiupload/MultiUploadArchiveTask.scala | 2 +- .../joex/process/AttachmentPreview.scala | 2 +- .../joex/process/ExtractArchive.scala | 2 +- .../docspell/joex/process/ItemData.scala | 21 +- .../docspell/joex/process/ItemHandler.scala | 27 +- .../docspell/joex/process/ProcessItem.scala | 4 + .../docspell/joex/process/ReProcessItem.scala | 40 +- .../docspell/joex/process/RunAddons.scala | 43 ++ .../docspell/joex/process/SetGivenData.scala | 2 +- .../docspell/joex/routes/JoexRoutes.scala | 9 +- .../src/main/resources/joex-openapi.yml | 37 ++ .../docspell/joexapi/client/JoexClient.scala | 10 +- .../docspell/notification/api/Event.scala | 4 +- .../src/main/resources/docspell-openapi.yml | 379 ++++++++++++++++ .../src/main/resources/reference.conf | 34 ++ .../docspell/restserver/RestAppImpl.scala | 15 +- .../docspell/restserver/RestServer.scala | 10 + .../conv/AddonValidationSupport.scala | 70 +++ .../restserver/http4s/Responses.scala | 4 + .../http4s/ThrowableResponseMapper.scala | 42 ++ .../routes/AddonArchiveRoutes.scala | 127 ++++++ .../restserver/routes/AddonRoutes.scala | 39 ++ .../routes/AddonRunConfigRoutes.scala | 110 +++++ .../restserver/routes/AddonRunRoutes.scala | 40 ++ .../restserver/routes/ItemMultiRoutes.scala | 6 +- .../restserver/routes/ItemRoutes.scala | 2 +- .../docspell/restserver/webapp/Flags.scala | 6 +- .../docspell/restserver/ws/Background.scala | 43 ++ .../docspell/restserver/ws/OutputEvent.scala | 24 + .../restserver/ws/OutputEventEncoder.scala | 18 + .../scala/docspell/scheduler/Context.scala | 6 + .../docspell/scheduler/PermanentError.scala | 26 ++ .../main/scala/docspell/scheduler/Task.scala | 5 + .../scheduler/usertask/UserTaskScope.scala | 3 + .../docspell/scheduler/impl/ContextImpl.scala | 13 + .../docspell/scheduler/impl/LogSink.scala | 1 + .../docspell/scheduler/impl/QueueLogger.scala | 5 +- .../scheduler/impl/SchedulerImpl.scala | 33 +- .../db/migration/h2/V1.36.0__addons.sql | 47 ++ .../db/migration/mariadb/V1.36.0__addons.sql | 47 ++ .../migration/postgresql/V1.36.0__addons.sql | 47 ++ .../docspell/store/file/FileUrlReader.scala | 64 +++ .../docspell/store/impl/DoobieMeta.scala | 11 +- .../docspell/store/queries/AttachedFile.scala | 31 ++ .../docspell/store/queries/QAttachment.scala | 41 +- .../store/records/AddonRunConfigData.scala | 155 +++++++ .../records/AddonRunConfigResolved.scala | 72 +++ .../store/records/RAddonArchive.scala | 184 ++++++++ .../store/records/RAddonRunConfig.scala | 99 ++++ .../store/records/RAddonRunConfigAddon.scala | 68 +++ .../records/RAddonRunConfigTrigger.scala | 62 +++ .../store/records/RAttachmentMeta.scala | 3 + .../store/records/RAttachmentPreview.scala | 20 +- .../scala/docspell/store/records/RItem.scala | 34 +- .../scala/docspell/store/records/RNode.scala | 20 +- .../scala/docspell/store/records/RUser.scala | 8 + project/Dependencies.scala | 3 + 166 files changed, 8181 insertions(+), 115 deletions(-) create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Context.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Directory.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Middleware.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/package.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala create mode 100644 modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/BackendCommands.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala create mode 100644 modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala create mode 100644 modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala create mode 100644 modules/common/src/main/scala/docspell/common/UrlMatcher.scala create mode 100644 modules/common/src/main/scala/docspell/common/UrlReader.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/ItemAction.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/Args.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/Env.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/SysCmd.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/SysExec.scala rename modules/common/src/main/scala/docspell/common/{ => util}/File.scala (91%) create mode 100644 modules/common/src/main/scala/docspell/common/util/Random.scala create mode 100644 modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala create mode 100644 modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala create mode 100644 modules/files/src/main/scala/docspell/files/FileSupport.scala create mode 100644 modules/files/src/test/resources/zip-dirs-one.zip create mode 100644 modules/files/src/test/resources/zip-dirs.zip create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/Result.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala create mode 100644 modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql create mode 100644 modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql create mode 100644 modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala diff --git a/build.sbt b/build.sbt index 2240a4e0..8156fc49 100644 --- a/build.sbt +++ b/build.sbt @@ -293,6 +293,15 @@ val openapiScalaSettings = Seq( field.copy(typeDef = TypeDef("DownloadState", Imports("docspell.common.DownloadState")) ) + case "addon-trigger-type" => + field => + field.copy(typeDef = + TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType")) + ) + case "addon-runner-type" => + field => + field + .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType"))) }) ) @@ -325,6 +334,7 @@ val common = project libraryDependencies ++= Dependencies.fs2 ++ Dependencies.circe ++ + Dependencies.circeGenericExtra ++ Dependencies.calevCore ++ Dependencies.calevCirce ) @@ -351,7 +361,7 @@ val files = project .in(file("modules/files")) .disablePlugins(RevolverPlugin) .settings(sharedSettings) - .withTestSettings + .withTestSettingsDependsOn(loggingScribe) .settings( name := "docspell-files", libraryDependencies ++= @@ -448,6 +458,19 @@ val notificationApi = project ) .dependsOn(common, loggingScribe) +val addonlib = project + .in(file("modules/addonlib")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .withTestSettingsDependsOn(loggingScribe) + .settings( + libraryDependencies ++= + Dependencies.fs2 ++ + Dependencies.circe ++ + Dependencies.circeYaml + ) + .dependsOn(common, files, loggingScribe) + val store = project .in(file("modules/store")) .disablePlugins(RevolverPlugin) @@ -469,7 +492,16 @@ val store = project libraryDependencies ++= Dependencies.testContainer.map(_ % Test) ) - .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe) + .dependsOn( + common, + addonlib, + query.jvm, + totp, + files, + notificationApi, + jsonminiq, + loggingScribe + ) val notificationImpl = project .in(file("modules/notification/impl")) @@ -647,7 +679,7 @@ val restapi = project openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, query.jvm, notificationApi, jsonminiq) + .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib) val joexapi = project .in(file("modules/joexapi")) @@ -667,7 +699,7 @@ val joexapi = project openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, loggingScribe) + .dependsOn(common, loggingScribe, addonlib) val backend = project .in(file("modules/backend")) @@ -683,6 +715,7 @@ val backend = project Dependencies.emil ) .dependsOn( + addonlib, store, notificationApi, joexapi, @@ -739,7 +772,7 @@ val config = project Dependencies.fs2 ++ Dependencies.pureconfig ) - .dependsOn(common, loggingApi, ftspsql, store) + .dependsOn(common, loggingApi, ftspsql, store, addonlib) // --- Application(s) @@ -946,6 +979,7 @@ val root = project ) .aggregate( common, + addonlib, loggingApi, loggingScribe, config, diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala new file mode 100644 index 00000000..3c2d6051 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.Zip + +final case class AddonArchive(url: LenientUri, name: String, version: String) { + def nameAndVersion: String = + s"$name-$version" + + def extractTo[F[_]: Async]( + reader: UrlReader[F], + directory: Path, + withSubdir: Boolean = true, + glob: Glob = Glob.all + ): F[Path] = { + val logger = docspell.logging.getLogger[F] + val target = + if (withSubdir) directory.absolute / nameAndVersion + else directory.absolute + + Files[F] + .exists(target) + .flatMap { + case true => target.pure[F] + case false => + Files[F].createDirectories(target) *> + reader(url) + .through(Zip.unzip(8192, glob)) + .through(Zip.saveTo(logger, target, moveUp = true)) + .compile + .drain + .as(target) + } + } + + /** Read meta either from the given directory or extract the url to find the metadata + * file to read + */ + def readMeta[F[_]: Async]( + urlReader: UrlReader[F], + directory: Option[Path] = None + ): F[AddonMeta] = + directory + .map(AddonMeta.findInDirectory[F]) + .getOrElse(AddonMeta.findInZip(urlReader(url))) +} + +object AddonArchive { + def read[F[_]: Async]( + url: LenientUri, + urlReader: UrlReader[F], + extractDir: Option[Path] = None + ): F[AddonArchive] = { + val addon = AddonArchive(url, "", "") + addon + .readMeta(urlReader, extractDir) + .map(m => addon.copy(name = m.meta.name, version = m.meta.version)) + } + + def dockerAndFlakeExists[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[(Boolean, Boolean)] = { + val files = Files[F] + def forPath(path: Path): F[(Boolean, Boolean)] = + (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled + + def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] = + data + .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix"))) + .collect { + case bin if bin.name == "Dockerfile" => (true, false) + case bin if bin.name == "flake.nix" => (false, true) + } + .compile + .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2)) + + archive.fold(forPath, forZip) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala new file mode 100644 index 00000000..08b3fd7b --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid +import cats.syntax.all._ + +case class AddonExecutionResult( + addonResults: List[AddonResult], + pure: Boolean +) { + def addonResult: AddonResult = addonResults.combineAll + def isFailure: Boolean = addonResult.isFailure + def isSuccess: Boolean = addonResult.isSuccess +} + +object AddonExecutionResult { + val empty: AddonExecutionResult = + AddonExecutionResult(Nil, false) + + def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult = + AddonExecutionResult( + a.addonResults ::: b.addonResults, + a.pure && b.pure + ) + + implicit val executionResultMonoid: Monoid[AddonExecutionResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala new file mode 100644 index 00000000..f13af4ef --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.Kleisli +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file._ + +import docspell.common.UrlReader +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonExecutor[F[_]] { + + def config: AddonExecutorConfig + + def execute(logger: Logger[F]): AddonExec[F] + + def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] = + execute(logger).run(in) +} + +object AddonExecutor { + + def apply[F[_]: Async]( + cfg: AddonExecutorConfig, + urlReader: UrlReader[F] + ): AddonExecutor[F] = + new AddonExecutor[F] with AddonLoggerExtension { + val config = cfg + + def execute(logger: Logger[F]): AddonExec[F] = + Kleisli { in => + for { + _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}") + ctx <- prepareDirectory( + logger, + in.baseDir, + in.outputDir, + in.cacheDir, + in.addons + ) + rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c)) + pure = ctx.foldl(true)((b, c) => b && c.meta.isPure) + } yield AddonExecutionResult(rs, pure) + } + + private def prepareDirectory( + logger: Logger[F], + baseDir: Path, + outDir: Path, + cacheDir: Path, + addons: List[AddonRef] + ): F[List[Context]] = + for { + addonsDir <- Directory.create(baseDir / "addons") + _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir) + _ <- Context + .userInputFile(baseDir) + .parent + .fold(().pure[F])(Files[F].createDirectories) + archives = addons.map(_.archive).distinctBy(_.url) + _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir") + mkCtxs <- archives.traverse { archive => + for { + _ <- logger.debug(s"Extracting $archive") + addonDir <- archive.extractTo(urlReader, addonsDir) + meta <- AddonMeta.findInDirectory(addonDir) + mkCtx = (ref: AddonRef) => + Context(ref, meta, baseDir, addonDir, outDir, cacheDir) + } yield archive.url -> mkCtx + } + ctxFactory = mkCtxs.toMap + res = addons.map(ref => ctxFactory(ref.archive.url)(ref)) + } yield res + + private def runAddon(logger: Logger[F], env: Env)( + ctx: Context + ): F[AddonResult] = + for { + _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}") + _ <- logger.trace("Storing user input into file") + _ <- Stream + .emit(ctx.addon.args) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(ctx.userInputFile, Flags.Write)) + .compile + .drain + + runner <- selectRunner(cfg, ctx.meta, ctx.addonDir) + result <- runner.run(logger, env, ctx) + } yield result + } + + def selectRunner[F[_]: Async]( + cfg: AddonExecutorConfig, + meta: AddonMeta, + addonDir: Path + ): F[AddonRunner[F]] = + for { + addonRunner <- meta.enabledTypes(Left(addonDir)) + // intersect on list retains order in first + possibleRunner = cfg.runner + .intersect(addonRunner) + .map(AddonRunner.forType[F](cfg)) + runner = possibleRunner match { + case Nil => + AddonRunner.failWith( + s"No runner available for addon config ${meta.runner} and config ${cfg.runner}." + ) + case list => + AddonRunner.firstSuccessful(list) + } + } yield runner +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala new file mode 100644 index 00000000..b4c29cb1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.AddonExecutorConfig._ +import docspell.common.Duration +import docspell.common.exec.{Args, SysCmd} + +case class AddonExecutorConfig( + runner: List[RunnerType], + runTimeout: Duration, + nspawn: NSpawn, + nixRunner: NixConfig, + dockerRunner: DockerConfig +) + +object AddonExecutorConfig { + + case class NSpawn( + enabled: Boolean, + sudoBinary: String, + nspawnBinary: String, + containerWait: Duration + ) { + val nspawnVersion = + SysCmd(nspawnBinary, Args.of("--version")).withTimeout(Duration.seconds(2)) + } + + case class NixConfig( + nixBinary: String, + buildTimeout: Duration + ) + + case class DockerConfig( + dockerBinary: String, + buildTimeout: Duration + ) { + def dockerBuild(imageName: String): SysCmd = + SysCmd(dockerBinary, "build", "-t", imageName, ".").withTimeout(buildTimeout) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala new file mode 100644 index 00000000..1b390985 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.logging.Logger + +trait AddonLoggerExtension { + + implicit final class LoggerAddonOps[F[_]](self: Logger[F]) { + private val addonName = "addon-name" + private val addonVersion = "addon-version" + + def withAddon(r: AddonArchive): Logger[F] = + self.capture(addonName, r.name).capture(addonVersion, r.version) + + def withAddon(r: Context): Logger[F] = + withAddon(r.addon.archive) + + def withAddon(m: AddonMeta): Logger[F] = + self.capture(addonName, m.meta.name).capture(addonVersion, m.meta.version) + } +} + +object AddonLoggerExtension extends AddonLoggerExtension diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala new file mode 100644 index 00000000..649b5d89 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.io.FileNotFoundException + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.Glob +import docspell.files.Zip + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.yaml.{parser => YamlParser} +import io.circe.{Decoder, Encoder} +import io.circe.{parser => JsonParser} + +case class AddonMeta( + meta: AddonMeta.Meta, + triggers: Option[Set[AddonTriggerType]], + args: Option[List[String]], + runner: Option[AddonMeta.Runner], + options: Option[AddonMeta.Options] +) { + + def nameAndVersion: String = + s"${meta.name}-${meta.version}" + + def parseResult: Boolean = + options.exists(_.collectOutput) + + def ignoreResult: Boolean = + !parseResult + + def isImpure: Boolean = + options.exists(_.isImpure) + + def isPure: Boolean = + options.forall(_.isPure) + + /** Returns a list of runner types that are possible to use for this addon. This is also + * inspecting the archive to return defaults when the addon isn't declaring it in the + * descriptor. + */ + def enabledTypes[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[List[RunnerType]] = + for { + filesExists <- AddonArchive.dockerAndFlakeExists(archive) + (dockerFileExists, flakeFileExists) = filesExists + + nixEnabled = runner.flatMap(_.nix).map(_.enable) match { + case Some(flag) => flag + case None => flakeFileExists + } + + dockerEnabled = runner.flatMap(_.docker).map(_.enable) match { + case Some(flag) => flag + case None => dockerFileExists + } + + trivialEnabled = runner.flatMap(_.trivial).exists(_.enable) + + result = RunnerType.all.filter(_.fold(nixEnabled, dockerEnabled, trivialEnabled)) + } yield result + +} + +object AddonMeta { + + def empty(name: String, version: String): AddonMeta = + AddonMeta(Meta(name, version, None), None, None, None, None) + + case class Meta(name: String, version: String, description: Option[String]) + case class Runner( + nix: Option[NixRunner], + docker: Option[DockerRunner], + trivial: Option[TrivialRunner] + ) + case class NixRunner(enable: Boolean) + case class DockerRunner(enable: Boolean, image: Option[String], build: Option[String]) + case class TrivialRunner(enable: Boolean, exec: String) + case class Options(networking: Boolean, collectOutput: Boolean) { + def isPure = !networking && collectOutput + def isImpure = networking + def isUseless = !networking && !collectOutput + def isUseful = networking || collectOutput + } + + object NixRunner { + implicit val jsonEncoder: Encoder[NixRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[NixRunner] = + deriveDecoder + } + + object DockerRunner { + implicit val jsonEncoder: Encoder[DockerRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[DockerRunner] = + deriveDecoder + } + + object TrivialRunner { + implicit val jsonEncoder: Encoder[TrivialRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[TrivialRunner] = + deriveDecoder + } + + object Runner { + implicit val jsonEncoder: Encoder[Runner] = + deriveEncoder + implicit val jsonDecoder: Decoder[Runner] = + deriveDecoder + } + + object Options { + implicit val jsonEncoder: Encoder[Options] = + deriveEncoder + implicit val jsonDecoder: Decoder[Options] = + deriveDecoder + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = + deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = + deriveDecoder + } + + implicit val jsonEncoder: Encoder[AddonMeta] = + deriveEncoder + + implicit val jsonDecoder: Decoder[AddonMeta] = + deriveDecoder + + def fromJsonString(str: String): Either[Throwable, AddonMeta] = + JsonParser.decode[AddonMeta](str) + + def fromJsonBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromJsonString) + .rethrow + + def fromYamlString(str: String): Either[Throwable, AddonMeta] = + YamlParser.parse(str).flatMap(_.as[AddonMeta]) + + def fromYamlBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromYamlString) + .rethrow + + def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = { + val logger = docspell.logging.getLogger[F] + val jsonFile = dir / "docspell-addon.json" + val yamlFile = dir / "docspell-addon.yaml" + val yamlFile2 = dir / "docspell-addon.yml" + + OptionT + .liftF(Files[F].exists(jsonFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading json addon file $jsonFile"))) + .flatMap(OptionT.whenF(_)(fromJsonBytes(Files[F].readAll(jsonFile)))) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile)))) + ) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile2)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile2"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile2)))) + ) + .getOrElseF( + Sync[F].raiseError( + new FileNotFoundException(s"No docspell-addon.{yaml|json} file found in $dir!") + ) + ) + } + + def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = { + val fail: F[AddonMeta] = Async[F].raiseError( + new FileNotFoundException( + s"No docspell-addon.{yaml|json} file found in zip!" + ) + ) + zipFile + .through(Zip.unzip(8192, Glob("**/docspell-addon.*"))) + .filter(bin => !bin.name.endsWith("/")) + .flatMap { bin => + if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data)) + else if (bin.extensionIn(Set("yaml", "yml"))) + Stream.eval(AddonMeta.fromYamlBytes(bin.data)) + else Stream.empty + } + .take(1) + .compile + .last + .flatMap(_.map(Sync[F].pure).getOrElse(fail)) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala new file mode 100644 index 00000000..37bc0106 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala @@ -0,0 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +case class AddonRef(archive: AddonArchive, args: String) diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala new file mode 100644 index 00000000..7efa9c26 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid + +import docspell.addons.out.AddonOutput + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Codec, Decoder, Encoder} + +sealed trait AddonResult { + def toEither: Either[Throwable, AddonOutput] + + def isSuccess: Boolean = toEither.isRight + def isFailure: Boolean = !isSuccess + + def cast: AddonResult = this +} + +object AddonResult { + + /** The addon was run successful, but decoding its stdout failed. */ + case class DecodingError(message: String) extends AddonResult { + def toEither = Left(new IllegalStateException(message)) + } + object DecodingError { + implicit val jsonEncoder: Encoder[DecodingError] = deriveEncoder + implicit val jsonDecoder: Decoder[DecodingError] = deriveDecoder + } + + def decodingError(message: String): AddonResult = + DecodingError(message) + + def decodingError(ex: Throwable): AddonResult = + DecodingError(ex.getMessage) + + /** Running the addon resulted in an invalid return code (!= 0). */ + case class ExecutionError(rc: Int) extends AddonResult { + def toEither = Left(new IllegalStateException(s"Exit code: $rc")) + } + + object ExecutionError { + implicit val jsonEncoder: Encoder[ExecutionError] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionError] = deriveDecoder + } + + def executionError(rc: Int): AddonResult = + ExecutionError(rc) + + /** The execution of the addon failed with an exception. */ + case class ExecutionFailed(error: Throwable) extends AddonResult { + def toEither = Left(error) + } + + object ExecutionFailed { + implicit val throwableCodec: Codec[Throwable] = + Codec.from( + Decoder[String].emap(str => Right(ErrorMessageThrowable(str))), + Encoder[String].contramap(_.getMessage) + ) + + implicit val jsonEncoder: Encoder[ExecutionFailed] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionFailed] = deriveDecoder + + private class ErrorMessageThrowable(msg: String) extends RuntimeException(msg) { + override def fillInStackTrace() = this + } + private object ErrorMessageThrowable { + def apply(str: String): Throwable = new ErrorMessageThrowable(str) + } + } + + def executionFailed(error: Throwable): AddonResult = + ExecutionFailed(error) + + /** The addon was run successfully and its output was decoded (if any). */ + case class Success(output: AddonOutput) extends AddonResult { + def toEither = Right(output) + } + + object Success { + implicit val jsonEncoder: Encoder[Success] = deriveEncoder + implicit val jsonDecoder: Decoder[Success] = deriveDecoder + } + + def success(output: AddonOutput): AddonResult = + Success(output) + + val empty: AddonResult = Success(AddonOutput.empty) + + def combine(a: AddonResult, b: AddonResult): AddonResult = + (a, b) match { + case (Success(o1), Success(o2)) => Success(AddonOutput.combine(o1, o2)) + case (Success(_), e) => e + case (e, Success(_)) => e + case _ => a + } + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("result").withKebabCaseConstructorNames + + implicit val jsonDecoder: Decoder[AddonResult] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonResult] = deriveConfiguredEncoder + + implicit val addonResultMonoid: Monoid[AddonResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala new file mode 100644 index 00000000..75efe4a2 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Applicative +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.runner._ +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonRunner[F[_]] { + def runnerType: List[RunnerType] + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] +} + +object AddonRunner { + def forType[F[_]: Async](cfg: AddonExecutorConfig)(rt: RunnerType) = + rt match { + case RunnerType.NixFlake => NixFlakeRunner[F](cfg) + case RunnerType.Docker => DockerRunner[F](cfg) + case RunnerType.Trivial => TrivialRunner[F](cfg) + } + + def failWith[F[_]](errorMsg: String)(implicit F: Applicative[F]): AddonRunner[F] = + pure(AddonResult.executionFailed(new Exception(errorMsg))) + + def pure[F[_]: Applicative](result: AddonResult): AddonRunner[F] = + new AddonRunner[F] { + val runnerType = Nil + + def run(logger: Logger[F], env: Env, ctx: Context) = + Applicative[F].pure(result) + } + + def firstSuccessful[F[_]: Sync](runners: List[AddonRunner[F]]): AddonRunner[F] = + runners match { + case Nil => failWith("No runner available!") + case a :: Nil => a + case _ => + new AddonRunner[F] { + val runnerType = runners.flatMap(_.runnerType).distinct + + def run(logger: Logger[F], env: Env, ctx: Context) = + Stream + .emits(runners) + .evalTap(r => + logger.info( + s"Attempt to run addon ${ctx.meta.nameAndVersion} with runner ${r.runnerType}" + ) + ) + .evalMap(_.run(logger, env, ctx)) + .flatMap { + case r @ AddonResult.Success(_) => Stream.emit(r.cast.some) + case r @ AddonResult.ExecutionFailed(ex) => + if (ctx.meta.isPure) { + logger.stream + .warn(ex)(s"Addon runner failed, try next.") + .as(r.cast.some) + } else { + logger.stream.warn(ex)(s"Addon runner failed!").as(None) + } + case r @ AddonResult.ExecutionError(rc) => + if (ctx.meta.isPure) { + logger.stream + .warn(s"Addon runner returned non-zero: $rc. Try next.") + .as(r.cast.some) + } else { + logger.stream.warn(s"Addon runner returned non-zero: $rc!").as(None) + } + case AddonResult.DecodingError(message) => + // Don't retry as it is very unlikely that the output differs using another runner + // This is most likely a bug in the addon + logger.stream + .warn( + s"Error decoding the output of the addon ${ctx.meta.nameAndVersion}: $message. Stopping here. This is likely a bug in the addon." + ) + .as(None) + } + .unNoneTerminate + .takeThrough(_.isFailure) + .compile + .last + .flatMap { + case Some(r) => r.pure[F] + case None => + AddonResult + .executionFailed(new NoSuchElementException("No runner left :(")) + .pure[F] + } + } + } + + def firstSuccessful[F[_]: Sync]( + runner: AddonRunner[F], + runners: AddonRunner[F]* + ): AddonRunner[F] = + firstSuccessful(runner :: runners.toList) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala new file mode 100644 index 00000000..85438f09 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList + +import io.circe.{Decoder, Encoder} + +sealed trait AddonTriggerType { + def name: String +} + +object AddonTriggerType { + + /** The final step when processing an item. */ + case object FinalProcessItem extends AddonTriggerType { + val name = "final-process-item" + } + + /** The final step when reprocessing an item. */ + case object FinalReprocessItem extends AddonTriggerType { + val name = "final-reprocess-item" + } + + /** Running periodically based on a schedule. */ + case object Scheduled extends AddonTriggerType { + val name = "scheduled" + } + + /** Running (manually) on some existing item. */ + case object ExistingItem extends AddonTriggerType { + val name = "existing-item" + } + + val all: NonEmptyList[AddonTriggerType] = + NonEmptyList.of(FinalProcessItem, FinalReprocessItem, Scheduled, ExistingItem) + + def fromString(str: String): Either[String, AddonTriggerType] = + all + .find(e => e.name.equalsIgnoreCase(str)) + .toRight(s"Invalid addon trigger type: $str") + + def unsafeFromString(str: String): AddonTriggerType = + fromString(str).fold(sys.error, identity) + + implicit val jsonEncoder: Encoder[AddonTriggerType] = + Encoder.encodeString.contramap(_.name) + + implicit val jsonDecoder: Decoder[AddonTriggerType] = + Decoder.decodeString.emap(fromString) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Context.scala b/modules/addonlib/src/main/scala/docspell/addons/Context.scala new file mode 100644 index 00000000..1c215e24 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Context.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import fs2.io.file.Path + +import docspell.common._ +import docspell.common.exec.{Args, Env, SysCmd} + +/** Context a list of addons is executed in. + * + * Each addon has its own `addonDir`, but all share the same `baseDir` in one run. + */ +case class Context( + addon: AddonRef, + meta: AddonMeta, + baseDir: Path, + addonDir: Path, + outputDir: Path, + cacheDir: Path +) { + def userInputFile = Context.userInputFile(baseDir) + def tempDir = Context.tempDir(baseDir) + + private[addons] def addonCommand( + binary: String, + timeout: Duration, + relativeToBase: Boolean, + outputDir: Option[String], + cacheDir: Option[String] + ): SysCmd = { + val execBin = Option + .when(relativeToBase)(binary) + .getOrElse((baseDir / binary).toString) + + val input = Option + .when(relativeToBase)(baseDir.relativize(userInputFile)) + .getOrElse(userInputFile) + + val allArgs = + Args(meta.args.getOrElse(Nil)).append(input) + val envAddonDir = Option + .when(relativeToBase)(baseDir.relativize(addonDir)) + .getOrElse(addonDir) + val envTmpDir = Option + .when(relativeToBase)(baseDir.relativize(tempDir)) + .getOrElse(tempDir) + val outDir = outputDir.getOrElse(this.outputDir.toString) + val cache = cacheDir.getOrElse(this.cacheDir.toString) + val moreEnv = + Env.of( + "ADDON_DIR" -> envAddonDir.toString, + "TMPDIR" -> envTmpDir.toString, + "TMP_DIR" -> envTmpDir.toString, + "OUTPUT_DIR" -> outDir, + "CACHE_DIR" -> cache + ) + + SysCmd(execBin, allArgs).withTimeout(timeout).addEnv(moreEnv) + } +} + +object Context { + def userInputFile(base: Path): Path = + base / "arguments" / "user-input" + def tempDir(base: Path): Path = + base / "temp" +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Directory.scala b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala new file mode 100644 index 00000000..3b72ac4c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import cats.{Applicative, Monad} +import fs2.io.file.{Files, Path, PosixPermissions} + +object Directory { + + def create[F[_]: Files: Applicative](dir: Path): F[Path] = + Files[F] + .createDirectories(dir, PosixPermissions.fromOctal("777")) + .as(dir) + + def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] = + (dir :: dirs.toList).traverse_(Files[F].createDirectories(_)) + + def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + List( + Files[F].isDirectory(dir), + Files[F].list(dir).take(1).compile.last.map(_.isDefined) + ).sequence.map(_.forall(identity)) + + def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + nonEmpty(dir).map(b => !b) + + def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + d <- mkTemp(parent, prefix) + } yield d + + def temp2[F[_]: Files]( + parent: Path, + prefix1: String, + prefix2: String + ): Resource[F, (Path, Path)] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + a <- mkTemp(parent, prefix1) + b <- mkTemp(parent, prefix2) + } yield (a, b) + + def createTemp[F[_]: Files: Monad]( + parent: Path, + prefix: String + ): F[Path] = + for { + _ <- Files[F].createDirectories(parent) + d <- mkTemp_(parent, prefix) + } yield d + + private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + Files[F] + .tempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) + + private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] = + Files[F] + .createTempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala new file mode 100644 index 00000000..bd5fb7cc --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect.Resource +import fs2.io.file.{Files, Path} + +import docspell.common.exec.Env + +case class InputEnv( + addons: List[AddonRef], + baseDir: Path, + outputDir: Path, + cacheDir: Path, + env: Env +) { + def addEnv(key: String, value: String): InputEnv = + copy(env = env.add(key, value)) + + def addEnv(vp: (String, String)*): InputEnv = + copy(env = env.addAll(vp.toMap)) + + def addEnv(vm: Map[String, String]): InputEnv = + copy(env = env ++ Env(vm)) + + def withTempBase[F[_]: Files]: Resource[F, InputEnv] = + Directory.temp(baseDir, "addon-").map(path => copy(baseDir = path)) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala new file mode 100644 index 00000000..4ca1a9fb --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monad +import cats.data.Kleisli +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.Files + +trait Middleware[F[_]] extends (AddonExec[F] => AddonExec[F]) { self => + + def >>(next: Middleware[F]): Middleware[F] = + Middleware(self.andThen(next)) +} + +object Middleware { + def apply[F[_]](f: AddonExec[F] => AddonExec[F]): Middleware[F] = + a => f(a) + + def identity[F[_]]: Middleware[F] = Middleware(scala.Predef.identity) + + /** Uses a temporary base dir that is removed after execution. Use this as the last + * layer! + */ + def ephemeralRun[F[_]: Files: Sync]: Middleware[F] = + Middleware(a => Kleisli(_.withTempBase.use(a.run))) + + /** Prepare running an addon */ + def prepare[F[_]: Monad]( + prep: Kleisli[F, InputEnv, InputEnv] + ): Middleware[F] = + Middleware(a => Kleisli(in => prep.run(in).flatMap(a.run))) + + def postProcess[F[_]: Monad]( + post: Kleisli[F, AddonExecutionResult, Unit] + ): Middleware[F] = + Middleware(_.flatMapF(r => post.map(_ => r).run(r))) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala new file mode 100644 index 00000000..adf343e1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList +import cats.syntax.all._ + +import io.circe.{Decoder, Encoder} + +sealed trait RunnerType { + def name: String + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A +} +object RunnerType { + case object NixFlake extends RunnerType { + val name = "nix-flake" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = nixFlake + } + case object Docker extends RunnerType { + val name = "docker" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = docker + } + case object Trivial extends RunnerType { + val name = "trivial" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = trivial + } + + val all: NonEmptyList[RunnerType] = + NonEmptyList.of(NixFlake, Docker, Trivial) + + def fromString(str: String): Either[String, RunnerType] = + all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid runner value: $str") + + def unsafeFromString(str: String): RunnerType = + fromString(str).fold(sys.error, identity) + + def fromSeparatedString(str: String): Either[String, List[RunnerType]] = + str.split("[\\s,]+").toList.map(_.trim).traverse(fromString) + + implicit val jsonDecoder: Decoder[RunnerType] = + Decoder[String].emap(RunnerType.fromString) + + implicit val jsonEncoder: Encoder[RunnerType] = + Encoder[String].contramap(_.name) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala new file mode 100644 index 00000000..143e3dac --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.kernel.Monoid + +import docspell.common.bc.BackendCommand + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Decoded stdout result from executing an addon. */ +case class AddonOutput( + commands: List[BackendCommand] = Nil, + files: List[ItemFile] = Nil, + newItems: List[NewItem] = Nil +) + +object AddonOutput { + val empty: AddonOutput = AddonOutput() + + def combine(a: AddonOutput, b: AddonOutput): AddonOutput = + AddonOutput(a.commands ++ b.commands, a.files ++ b.files) + + implicit val addonResultMonoid: Monoid[AddonOutput] = + Monoid.instance(empty, combine) + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonDecoder: Decoder[AddonOutput] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonOutput] = deriveConfiguredEncoder + + def fromString(str: String): Either[Throwable, AddonOutput] = + io.circe.parser.decode[AddonOutput](str) + + def unsafeFromString(str: String): AddonOutput = + fromString(str).fold(throw _, identity) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala new file mode 100644 index 00000000..bd470613 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.FileSupport._ +import docspell.logging.Logger + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Addons can produce files in their output directory. These can be named here in order + * to do something with them. + * + * - textFiles will replace the extracted text with the contents of the file + * - pdfFiles will add/replace the converted pdf with the given file + * - previewImages will add/replace preview images + * - newFiles will be added as new attachments to the item + * + * Files must be referenced by attachment id. + */ +final case class ItemFile( + itemId: Ident, + textFiles: Map[String, String] = Map.empty, + pdfFiles: Map[String, String] = Map.empty, + previewImages: Map[String, String] = Map.empty, + newFiles: List[NewFile] = Nil +) { + def isEmpty: Boolean = + textFiles.isEmpty && pdfFiles.isEmpty && previewImages.isEmpty + + def nonEmpty: Boolean = !isEmpty + + def resolveTextFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.text("*"), textFiles) + + def resolvePdfFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.pdf, pdfFiles) + + def resolvePreviewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.image("*"), previewImages) + + def resolveNewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(NewFile, Path)]] = + newFiles.traverseFilter(nf => + nf.resolveFile(logger, outputDir).map(_.map(p => (nf, p))) + ) + + private def resolveFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path, + mime: MimeType, + files: Map[String, String] + ): F[List[(String, Path)]] = { + val allFiles = + files.toList.map(t => t._1 -> outputDir / t._2) + + allFiles.traverseFilter { case (key, file) => + OptionT(file.detectMime) + .flatMapF(fileType => + if (mime.matches(fileType)) (key -> file).some.pure[F] + else + logger + .warn( + s"File $file provided as ${mime.asString} file, but was recognized as ${fileType.asString}. Ignoring it." + ) + .as(None: Option[(String, Path)]) + ) + .value + } + } +} + +object ItemFile { + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonEncoder: Encoder[ItemFile] = deriveConfiguredEncoder + implicit val jsonDecoder: Decoder[ItemFile] = deriveConfiguredDecoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala new file mode 100644 index 00000000..4807fe9c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.effect.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewFile.Meta +import docspell.common.ProcessItemArgs.ProcessMeta +import docspell.common.{Ident, Language} +import docspell.logging.Logger + +import io.circe.Codec +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.deriveConfiguredCodec +import io.circe.generic.semiauto.deriveCodec + +case class NewFile(metadata: Meta = Meta.empty, file: String) { + + def resolveFile[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[Option[Path]] = { + val target = outputDir / file + Files[F] + .exists(target) + .flatMap(flag => + if (flag) target.some.pure[F] + else logger.warn(s"File not found: $file").as(Option.empty) + ) + } +} + +object NewFile { + + case class Meta( + language: Option[Language], + skipDuplicate: Option[Boolean], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessMeta( + cid: Ident, + itemId: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessMeta = + ProcessMeta( + collective = cid, + itemId = Some(itemId), + language = language.orElse(collLang).getOrElse(Language.English), + direction = None, + sourceAbbrev = sourceAbbrev, + folderId = None, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = None, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + val empty = Meta(None, None, None) + implicit val jsonCodec: Codec[Meta] = deriveCodec + } + + implicit val jsonConfig: Configuration = Configuration.default.withDefaults + + implicit val jsonCodec: Codec[NewFile] = deriveConfiguredCodec +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala new file mode 100644 index 00000000..c5511fd0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.Monad +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewItem.Meta +import docspell.common._ +import docspell.logging.Logger + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +case class NewItem(metadata: Option[Meta], files: List[String]) { + + def toProcessMeta( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + metadata + .getOrElse(Meta(None, None, None, None, None, None, None)) + .toProcessArgs(cid, collLang, sourceAbbrev) + + def resolveFiles[F[_]: Files: Monad]( + logger: Logger[F], + outputDir: Path + ): F[List[Path]] = { + val allFiles = + files.map(name => outputDir / name) + + allFiles.traverseFilter { file => + Files[F] + .exists(file) + .flatMap { + case true => file.some.pure[F] + case false => + logger + .warn(s"File $file doesn't exist. Ignoring it.") + .as(None) + } + } + } +} + +object NewItem { + + case class Meta( + language: Option[Language], + direction: Option[Direction], + folderId: Option[Ident], + source: Option[String], + skipDuplicate: Option[Boolean], + tags: Option[List[String]], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessArgs( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + ProcessItemArgs.ProcessMeta( + collective = cid, + itemId = None, + language = language.orElse(collLang).getOrElse(Language.English), + direction = direction, + sourceAbbrev = source.getOrElse(sourceAbbrev), + folderId = folderId, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = tags, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = deriveDecoder + } + + implicit val jsonDecoder: Decoder[NewItem] = deriveDecoder + implicit val jsonEncoder: Encoder[NewItem] = deriveEncoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/package.scala b/modules/addonlib/src/main/scala/docspell/addons/package.scala new file mode 100644 index 00000000..82a67232 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/package.scala @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell + +import cats.data.Kleisli + +package object addons { + + type AddonExec[F[_]] = Kleisli[F, InputEnv, AddonExecutionResult] + +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala new file mode 100644 index 00000000..48fa5b75 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.Applicative +import cats.effect.{Ref, Sync} +import cats.syntax.all._ +import fs2.Pipe + +trait CollectOut[F[_]] { + + def get: F[String] + + def append: Pipe[F, String, String] +} + +object CollectOut { + + def none[F[_]: Applicative]: CollectOut[F] = + new CollectOut[F] { + def get = "".pure[F] + def append = identity + } + + def buffer[F[_]: Sync]: F[CollectOut[F]] = + Ref + .of[F, Vector[String]](Vector.empty) + .map(buffer => + new CollectOut[F] { + override def get = + buffer.get.map(_.mkString("\n").trim) + + override def append = + _.evalTap(line => + if (line.trim.nonEmpty) buffer.update(_.appended(line)) else ().pure[F] + ) + } + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala new file mode 100644 index 00000000..b41d46e0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} + +/** Builder for a docker system command. */ +case class DockerBuilder( + dockerBinary: String, + subCmd: String, + timeout: Duration, + containerName: Option[String] = None, + env: Env = Env.empty, + mounts: Args = Args.empty, + network: Option[String] = Some("host"), + workingDir: Option[String] = None, + imageName: Option[String] = None, + cntCmd: Args = Args.empty +) { + def containerCmd(args: Args): DockerBuilder = + copy(cntCmd = args) + def containerCmd(args: Seq[String]): DockerBuilder = + copy(cntCmd = Args(args)) + + def imageName(name: String): DockerBuilder = + copy(imageName = Some(name)) + + def workDirectory(dir: String): DockerBuilder = + copy(workingDir = Some(dir)) + + def withDockerBinary(bin: String): DockerBuilder = + copy(dockerBinary = bin) + + def withSubCmd(cmd: String): DockerBuilder = + copy(subCmd = cmd) + + def withEnv(key: String, value: String): DockerBuilder = + copy(env = env.add(key, value)) + + def withEnv(moreEnv: Env): DockerBuilder = + copy(env = env ++ moreEnv) + + def privateNetwork(flag: Boolean): DockerBuilder = + if (flag) copy(network = Some("none")) + else copy(network = Some("host")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): DockerBuilder = { + val target = cntDir.getOrElse(hostDir.toString) + val ro = Option.when(readOnly)(",readonly").getOrElse("") + val opt = s"type=bind,source=$hostDir,target=$target${ro}" + copy(mounts = mounts.append("--mount", opt)) + } + + def withName(containerName: String): DockerBuilder = + copy(containerName = Some(containerName)) + + def build: SysCmd = + SysCmd(dockerBinary, buildArgs).withTimeout(timeout) + + private def buildArgs: Args = + Args + .of(subCmd) + .append("--rm") + .option("--name", containerName) + .append(mounts) + .option("--network", network) + .append(env.mapConcat((k, v) => List("--env", s"${k}=${v}"))) + .option("-w", workingDir) + .appendOpt(imageName) + .append(cntCmd) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala new file mode 100644 index 00000000..a20ab5a0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonExecutorConfig.DockerConfig +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Env, SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +final class DockerRunner[F[_]: Async](cfg: DockerRunner.Config) extends AddonRunner[F] { + + val runnerType = List(RunnerType.Docker) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = for { + _ <- OptionT.whenF(requireBuild(ctx))(build(logger, ctx)).value + suffix <- Random[F].string(4) + cmd = createDockerCommand(env, ctx, suffix) + result <- RunnerUtil.runAddonCommand(logger, cmd, ctx) + } yield result + + def createDockerCommand( + env: Env, + ctx: Context, + suffix: String + ): SysCmd = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonArgs = + ctx.addonCommand( + "", + Duration.zero, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + + DockerBuilder(cfg.docker.dockerBinary, "run", cfg.timeout) + .withName(ctx.meta.nameAndVersion + "-" + suffix) + .withEnv(env) + .withEnv(addonArgs.env) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .workDirectory("/mnt/work") + .privateNetwork(ctx.meta.isPure) + .imageName(imageName(ctx)) + .containerCmd(addonArgs.args) + .build + } + + def build(logger: Logger[F], ctx: Context): F[Unit] = + for { + _ <- logger.info(s"Building docker image for addon ${ctx.meta.nameAndVersion}") + cmd = cfg.docker.dockerBuild(imageName(ctx)) + _ <- SysExec(cmd, logger, ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "docker build")) + .use(_.waitFor()) + _ <- logger.info(s"Docker image built successfully") + } yield () + + private def requireBuild(ctx: Context) = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .isEmpty + + private def imageName(ctx: Context): String = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .getOrElse(s"${ctx.meta.meta.name}:latest") +} + +object DockerRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): DockerRunner[F] = + new DockerRunner[F](Config(cfg.dockerRunner, cfg.runTimeout)) + + case class Config(docker: DockerConfig, timeout: Duration) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala new file mode 100644 index 00000000..accd2059 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.exec.{Args, Env, SysCmd} + +case class NSpawnBuilder( + child: SysCmd, + chroot: Path, + spawnBinary: String = "systemd-nspawn", + sudoBinary: String = "sudo", + args: Args = Args.empty, + env: Env = Env.empty +) { + + def withNSpawnBinary(bin: String): NSpawnBuilder = + copy(spawnBinary = bin) + + def withSudoBinary(bin: String): NSpawnBuilder = + copy(sudoBinary = bin) + + def withEnv(key: String, value: String): NSpawnBuilder = + copy(args = args.append(s"--setenv=$key=$value")) + + def withEnvOpt(key: String, value: Option[String]): NSpawnBuilder = + value.map(v => withEnv(key, v)).getOrElse(this) + + def withName(containerName: String): NSpawnBuilder = + copy(args = args.append(s"--machine=$containerName")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): NSpawnBuilder = { + val bind = if (readOnly) "--bind-ro" else "--bind" + val target = cntDir.map(dir => s":$dir").getOrElse("") + copy(args = args.append(s"${bind}=${hostDir}${target}")) + } + + def workDirectory(dir: String): NSpawnBuilder = + copy(args = args.append(s"--chdir=$dir")) + + def portMap(port: Int): NSpawnBuilder = + copy(args = args.append("-p", port.toString)) + + def privateNetwork(flag: Boolean): NSpawnBuilder = + if (flag) copy(args = args.append("--private-network")) + else this + + def build: SysCmd = + SysCmd( + program = if (sudoBinary.nonEmpty) sudoBinary else spawnBinary, + args = buildArgs, + timeout = child.timeout, + env = env + ) + + private def buildArgs: Args = + Args + .of("--private-users=identity") // can't use -U because need writeable bind mounts + .append("--notify-ready=yes") + .append("--ephemeral") + .append("--as-pid2") + .append("--console=pipe") + .append("--no-pager") + .append("--bind-ro=/bin") + .append("--bind-ro=/usr/bin") + .append("--bind-ro=/nix/store") + .append(s"--directory=$chroot") + .append(args) + .append(child.env.map((n, v) => s"--setenv=$n=$v")) + .prependWhen(sudoBinary.nonEmpty)(spawnBinary) + .append("--") + .append(child.program) + .append(child.args) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala new file mode 100644 index 00000000..ce12c73c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala @@ -0,0 +1,123 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.addons.AddonExecutorConfig.{NSpawn, NixConfig} +import docspell.addons._ +import docspell.addons.runner.NixFlakeRunner.PreCtx +import docspell.common.Duration +import docspell.common.exec._ +import docspell.logging.Logger + +final class NixFlakeRunner[F[_]: Async](cfg: NixFlakeRunner.Config) + extends AddonRunner[F] { + + val runnerType = List(RunnerType.NixFlake) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] = + prepare(logger, ctx) + .flatMap { preCtx => + if (preCtx.nspawnEnabled) runInContainer(logger, env, preCtx, ctx) + else runOnHost(logger, env, preCtx, ctx) + } + + def prepare(logger: Logger[F], ctx: Context): F[PreCtx] = + for { + _ <- logger.info(s"Prepare addon ${ctx.meta.nameAndVersion} for executing via nix") + _ <- logger.debug(s"Building with nix build") + _ <- SysExec(cfg.nixBuild, logger, workdir = ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "nix build")) + .use(_.waitFor()) + bin <- findFile(ctx.addonDir / "result" / "bin", ctx.addonDir / "result") + _ <- logger.debug(s"Build done, found binary: $bin") + _ <- logger.debug(s"Checking for systemd-nspawn…") + cnt <- checkContainer(logger) + _ <- + if (cnt) + logger.debug(s"Using systemd-nspawn to run addon in a container.") + else + logger.info(s"Running via systemd-nspawn is disabled in the config file") + } yield PreCtx(cnt, ctx.baseDir.relativize(bin)) + + private def checkContainer(logger: Logger[F]): F[Boolean] = + if (!cfg.nspawn.enabled) false.pure[F] + else RunnerUtil.checkContainer(logger, cfg.nspawn) + + private def runOnHost( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = + SysCmd(preCtx.binary.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + + private def runInContainer( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = SysCmd(preCtx.binary.toString, Args.empty) + .withTimeout(cfg.timeout) + .addEnv(env) + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + } + + /** Find first file, try directories in given order. */ + private def findFile(firstDir: Path, more: Path*): F[Path] = { + val fail: F[Path] = Sync[F].raiseError( + new NoSuchElementException( + s"No file found to execute in ${firstDir :: more.toList}" + ) + ) + + Stream + .emits(more) + .cons1(firstDir) + .flatMap(dir => + Files[F] + .list(dir) + .evalFilter(p => Files[F].isDirectory(p).map(!_)) + .take(1) + ) + .take(1) + .compile + .last + .flatMap(_.fold(fail)(Sync[F].pure)) + } +} + +object NixFlakeRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): NixFlakeRunner[F] = + new NixFlakeRunner[F](Config(cfg.nixRunner, cfg.nspawn, cfg.runTimeout)) + + case class Config( + nix: NixConfig, + nspawn: NSpawn, + timeout: Duration + ) { + + val nixBuild = + SysCmd(nix.nixBinary, Args.of("build")).withTimeout(nix.buildTimeout) + + val nspawnVersion = nspawn.nspawnVersion + } + + case class PreCtx(nspawnEnabled: Boolean, binary: Path) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala new file mode 100644 index 00000000..e404b442 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala @@ -0,0 +1,171 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect.{Async, Sync} +import cats.syntax.all._ +import fs2.Pipe +import fs2.io.file.Files + +import docspell.addons._ +import docspell.addons.out.AddonOutput +import docspell.common.exec.{SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +import io.circe.{parser => JsonParser} + +private[addons] object RunnerUtil { + + /** Run the given `cmd` on this machine. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runDirectly[F[_]: Async]( + logger: Logger[F], + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val addonCmd = ctx + .addonCommand(cmd.program, cmd.timeout, relativeToBase = false, None, None) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + runAddonCommand(logger, addonCmd, ctx) + } + + /** Run the given `cmd` inside a container via systemd-nspawn. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runInContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn, + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonCmd = ctx + .addonCommand( + cmd.program, + cmd.timeout, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + + val chroot = ctx.baseDir / "cnt-root" + val nspawn = NSpawnBuilder(addonCmd, chroot) + .withNSpawnBinary(cfg.nspawnBinary) + .withSudoBinary(cfg.sudoBinary) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .workDirectory("/mnt/work") + .withEnv("XDG_RUNTIME_DIR", "/mnt/work") + .privateNetwork(ctx.meta.isPure) + + for { + suffix <- Random[F].string(4) + _ <- List(chroot).traverse_(Files[F].createDirectories) + res <- runAddonCommand( + logger, + nspawn.withName(ctx.meta.nameAndVersion + "-" + suffix).build, + ctx + ) + // allow some time to unregister the current container + // only important when same addons are called in sequence too fast + _ <- Sync[F].sleep(cfg.containerWait.toScala) + } yield res + } + + private def procPipe[F[_]]( + p: String, + ctx: Context, + collect: CollectOut[F], + logger: Logger[F] + ): Pipe[F, String, Unit] = + _.through(collect.append) + .map(line => s">> [${ctx.meta.nameAndVersion} ($p)] $line") + .evalMap(logger.debug(_)) + + /** Runs the external command that is executing the addon. + * + * If the addons specifies to collect its output, the stdout is parsed as json and + * decoded into [[AddonOutput]]. + */ + def runAddonCommand[F[_]: Async]( + logger: Logger[F], + cmd: SysCmd, + ctx: Context + ): F[AddonResult] = + for { + stdout <- + if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F] + else CollectOut.none[F].pure[F] + cmdResult <- SysExec(cmd, logger, ctx.baseDir.some) + .flatMap( + _.consumeOutputs( + procPipe("out", ctx, stdout, logger), + procPipe("err", ctx, CollectOut.none[F], logger) + ) + ) + .use(_.waitFor()) + .attempt + addonResult <- cmdResult match { + case Right(rc) if rc != 0 => + for { + _ <- logger.error( + s"Addon ${ctx.meta.nameAndVersion} returned non-zero: $rc" + ) + } yield AddonResult.executionError(rc) + + case Right(_) => + for { + _ <- logger.debug(s"Addon ${ctx.meta.nameAndVersion} executed successfully!") + out <- stdout.get + _ <- logger.debug(s"Addon stdout: $out") + result = Option + .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)( + JsonParser + .decode[AddonOutput](out) + .fold(AddonResult.decodingError, AddonResult.success) + ) + .getOrElse(AddonResult.empty) + } yield result + + case Left(ex) => + logger + .error(ex)(s"Executing external command failed!") + .as(AddonResult.executionFailed(ex)) + } + } yield addonResult + + /** Check whether `systemd-nspawn` is available on this machine. */ + def checkContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn + ): F[Boolean] = + for { + rc <- SysExec(cfg.nspawnVersion, logger) + .flatMap(_.logOutputs(logger, "nspawn")) + .use(_.waitFor()) + _ <- + OptionT + .whenF(rc != 0)( + logger.warn( + s"No systemd-nspawn found! Addon is not executed inside a container." + ) + ) + .value + } yield rc == 0 +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala new file mode 100644 index 00000000..31ac2694 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.kernel.Monoid +import cats.syntax.all._ +import fs2.io.file.PosixPermission._ +import fs2.io.file.{Files, PosixPermissions} + +import docspell.addons.AddonExecutorConfig.NSpawn +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} +import docspell.logging.Logger + +final class TrivialRunner[F[_]: Async](cfg: TrivialRunner.Config) extends AddonRunner[F] { + private val sync = Async[F] + private val files = Files[F] + implicit val andMonoid: Monoid[Boolean] = Monoid.instance[Boolean](true, _ && _) + + private val executeBits = PosixPermissions( + OwnerExecute, + OwnerRead, + OwnerWrite, + GroupExecute, + GroupRead, + OthersExecute, + OthersRead + ) + + val runnerType = List(RunnerType.Trivial) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = { + val binaryPath = ctx.meta.runner + .flatMap(_.trivial) + .map(_.exec) + .map(bin => ctx.addonDir / bin) + + binaryPath match { + case None => + sync.raiseError(new IllegalStateException("No executable specified in addon!")) + + case Some(file) => + val bin = ctx.baseDir.relativize(file) + val cmd = SysCmd(bin.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + + val withNSpawn = + OptionT + .whenF(cfg.nspawn.enabled)(RunnerUtil.checkContainer(logger, cfg.nspawn)) + .getOrElse(false) + + files.setPosixPermissions(file, executeBits).attempt *> + withNSpawn.flatMap { + case true => + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + case false => + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + } + } +} + +object TrivialRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): TrivialRunner[F] = + new TrivialRunner[F](Config(cfg.nspawn, cfg.runTimeout)) + + case class Config(nspawn: NSpawn, timeout: Duration) +} diff --git a/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip b/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip new file mode 100644 index 0000000000000000000000000000000000000000..20482861376733b293b75844b34dced90b3b330d GIT binary patch literal 7634 zcmb7p1yodB)b_vtBHg7DBPcl_-Q6&BO2Z62bVx`jLxTt-0us_9Eg&TzASn&f4NA8N z67u2e{~EmN|9xlHxif3lI?vtro@ej#?7h{LfM{5NuOr4XRqLN0|L;NyxD0?gTDUsF z?Csg0?r^vln;8`9=)kV6g$_X3A%UB#{jwHM1x<|VeC^js^QX07D;qZ(D+fmx*mo2c zn5(W*1L$(F$t?)rRBE6wx*Q4KVp9qFUN67hNlQLK5OcijGBokmG`%f2OS^>VvAVYwxA zXt;Ip51xFa4UyY;wJFkM)P*`Vhv&wdX~AvCso`32+YcB~H#isja1@cwu$7ae4<~y{ zTnXevLV2zyr_@Q0dpw}up|cYLid~6WG8siT-w>G;3t$@K>t-sDJ0QAQ0u?+Y0X;~d zlL3tgZQa3_oX0$TL&4fbh0s_r%^bt>N_8C8W06vtX#kr+RowE1X78}xwVKZgQn_5G z8=KGRP2wt_Q9GOYlDWzZ+EBdgC^Ck>HIWzNcozJ?H4*JB{h6fxXX|p^I_!$R`~v52 zGErm0SB=jt4QYIzHQU%(EYB)}!}Aqx%~rq4l9B0vKCU_&nm1CMG|3tygl5%|U9iQN@A6E_!i+IH_sxcKEztXg|s(_AQ)EXOE`R2niqk;dJEz zM9qh{=5;l7`o`?T+d!4+7!TBjB(Yx!AXsfyB5_VGne$2gCxU6nNJv9uz;Nhyerf7B zDJyjJYGNv|f#>G2I@(!v02$H<f7(5u+4S8g#-KNn{S zTa{=@kbE&UIp9|-Owvq>uA>70&Da3I%|BW3-EaQ!7_b-I{<}E{9f;!sFG1b0t}@?a zqu1&I)2kMxArAOYP@gC{T0VQ&A`8;?dvmLgW`<8{TxSwYhB+HbCED}KYwwwmH&vo# zPcYHM`n{Wv8Fyz|WHyD4Y%Omr_;|lhSllqwxIx1?He2TG=PK9M4^xRKeMR_mD!AQZ zB3kq#ELHU&XY2l1z`Ni;axu?_cFEoj*-EI<$t7&*8C1NXyIIGKhbz)jp+Q_P_Cg%) zUl-e(6{dFL_36CsQq&*oyL0V%lm8)0m``UQUdw)@c>E(wv?Z=K6oCaXq^QEqYp#0K z^j+~{OJ>RFoq)6)2iK4&{~+Y#-kW{+xY*Pl{dH5d9J#CN!x zs_3?+(K_x!_-4Jtls_PZShun#q!{@-I}~t`B#P4aEdmS~LPSR&zwPNg;;0HMHn0ov z$Mv$$rJZVzIeIm#i*UONi9bvtvVGghE{*Q)`J@Jdi55-5exeA#6qM$Y@_PYtw_eRt z*w3JndK^n4O=M46m%p?elZB#wpGotE@=S!lkUE^;fK3MR5GW<{nPvM*cWZQy>rHl^ zS(``Kck`lr+-VhGt}YlGxMNFXDul-jZ8Yy740+XFl>97-i__x z#_AVFmmat-Ip5D<+_vhZiJxH~%u;?&r-F7QWn8}S+V}$mU%1gRq*r#pn(Bp`AG;1{ z{DsrYfO)GqLNAaB8wJVGF6jq^>KjM!dugSBFl&F{y&~^%TOHCR$@g5%&Go&NL)rFx zQS)9iHhn@x@9xU&b%r|+?LCPm+dG*ZEv+AyMo;|ehIBmP4I{dN!6-S)j+Ud zz@+WG-G=VP-k;cI?zEG|sEC2?T>GUd2x*Y^C70!eQ*NKoWkqb&L8-KL2Tt?-Bjf6c z!m7eL-*7xAWQo@@v19QAEOxqtO@Ce22)`L$Un9DoIGZGtKQzvd%99}xsUTay2^KnyxkL;6( z&luaBxHa91$lLJ+Gb8m4^;S8G|{(ihrBTP^lebx5fy z>vl;KFy)J}m&O9mS`7?Mo^UEUwYYoP@*Do{+hqzF1pO(CZrc*|9>s=BN2HMv{yJtn4fZA+<^k59 zbdIXZdmB}F^J;U-@}FnU?KY3Rf9m-LgaJDVqb2c=F|)}tLXvohysBbaE~{-K5ka7;eO*)Anj7{wOeb_sdknrfbha z&*r9avfk2x-`HJNjFvucE*g?%4waBL!l$Jkn}_XsPmlfgAlXMTlCQ!9%z zORea8lb_m}lI@rGPN9o0D2QXjUA9=*f$8^$<}2?s21+24C34FY(E=>o1A(ez5SbxZQQjG zr_DJ;Isqo+q&F5uVT11W=Dw>L*8Pdq++Mq+xph!#NZ}g-jM5-Q7f6@{$1KLcYc&}Z za~#T$XKqe5wqIguRwf-14uX$10_pWfaV<#MXgY3ybfw4jLk|{66HRh&`j}yAElwqs z06qk9?tlv>!Y|>fchk=r$wSFxGM!c9KGKtntVRdlopp5<69-=^DK0btswBt5*fb`jH`;c(r$-lVCv#kiQIni`A2hl& z0hFStmkD`6cFixFUfI;eq3zN`3MkrRGb7hPPojz{8d zFEOTSQ5H@6YgT;HdoOF!?_GGifpKY%Avw9C(?dA!G(J)-kS?02i-H%6F$W7LHXSsh zFHYEc)wS)`I4F_mQRBV#Q?qr~>`V#1Ln413s3at zQTZ4fhq59-NtzgSpsu6CR6?q!V=c?`cNnv%OW~uzSh8ME(aK{4W*KqNoYYVj@6GG$ zgU$r1`iitT7vc}^qWK4xu^*lZIt}P1Epo?F)S--}gI@ z4bt$-P50b&vmRE~@WPP+^aK)LHs8H18q~_VnfqeqR;Hk1kO^Mvqh6EV3_Mg3XOI)N zi7`c@$6|N<0)s-XdXEESRud9Ti`f z8)&xV@x0wkvAC5RT(%Y`o3rZ|4KAib=x#i7_Vel+Iw^>{J*11uVf#q|b$@3i-mVBl zr7>UrQWo=iC`mX~QG~IAJgo4c3Yq(3$1<)XPutD)(=WYCVy#w73?Es;#Lh(Z!#$Uq z>A{h{OUFGXzj}Y`J<9gobJhO_?JuJ5oA*1|-1+YOQJNj+qW@fFv0y0T>OOyE-;n_Un>EIqfNib(rfw}0Wcz6FMS>gNO;jBrn$ryd{H+phe z`7BQJj45x^i5Lf-#(#Psh(;<02TApbJfGi2`4 zX7|7;B6X^}-X+1%Ydp?9j57tY17r$GJGoEBh`j8w?;R(=n{hvs zqXk^g5ko<1VR>qZ1=ILapSzQ}@sQD$_Jf`!^&B(waDBm~%+ASIWdpVRku)0ywNp0# z2Dpi@GMYKf`Q0o~Vg~#$)4;gJ4S393HZ7X2uN5J%vjO2sLGUDHZz#A2g0yv>M{<@A z&JnwTr4qCelxqyAyP3>Vpa$a1sNF`OKFyL@o#+`UEVxDEgQ95L+b0!B6Iu_qRqoDP z1Et1^y|1X$@Q2b~%AH{y*KglQl4A0+*m^XJS5bzf9JdXMQg7fWlMWG)P0V^2c3g6l z?mvCMqd|85Kuq_Tu#KMNvJsVQ4g88>1I!(zfbU6e+su=YG?jAlgnX~hJ6EsC*i^B~ z){s~3c?hiBI79iDHvdgzDgUI>&k6J2WcFLKDYX604E3CL_2(1xxAory{?F8Nk7=}h zLrW06{gSSrO8KzRiG0V~YN`%`?TwLu>;lygq! z_V%;4HVYb^C(OF3RHw@o%0zAi@eJVyD1olLA}P^S^V$8>pYAOCxy?CwMA-hi$^QNJ zoj%XdvH(TI!q$o+zY_jn?p~?XO4Bnf{+yac&XdOr%}{jVjRSvcqsn);ProNj|A(z` zb+P!pSqxK#$*>y8)t9sV_aqmj2F z@XUdF+Er?E6`Dwm9!qvmP)g!=VGxz^~`8` zhES=sQl$_uum&2ZzZw{LtKvrSwOA!4L(4`=CrVVX^r~s}BW>A(IX)Z6hIkx){{yw5 z;YOq{w={Y4qFh>vA>QGsNl$ceM?h|qGjEM3m6MHTPr?9KSp}{t&a%Ks$)yBKj5{Br z%E()N{fi48^G2Rw-CFljzi;yH_OT8EDZ58@0%M9T1>05P_dA5()ddC2S-bgp2vM21 zOW%`W(w5P3@g{W2u`EI1BRIRiY?FL7ZTkFn+n*MXPSzLZ(QlaC9rRwgV-bRWCrNpF zO`v&l-g_mG7th);Z5o;ubY`&#e#<|8HRYSS4ql=^qewlzEX!qPd*OT%}t$Uyy?3YUd^8liMaJYOqd`n`glU`Z?T*;=_uCt-J4Dm zEK+I^frLF)UH1MweFpq3q`u6~M*a0sIroWg;PX z_c$n|4@6q5=+Ma2*k5NT)V@aJXio*e%ah17!d-mrP9`89>Jm-mhFfAKC_&xVg8iy# zxv^k-9MtGRr4MpOh@7h0E=v=3;XkC!a-zunWHpX8f>OU{Y)yd5BmnSUL>R0*4k9F_36&J*aiEsTNTherYoRfvxWxL25K^j+UA5jjfV47 zzv^>`6DCdNXoa)!yc`za>;6>DnEJ9BjLJKwZtfRB8#B?t2!1Y31ws~$kbz=xn(t7o zg9)*^b9F}sKc~kEJqyp7w|y4`v52R{AG0uA?O1RD#c9e0q1C?3;HIaN6}pC<-Q4%N zd+MVaQx6q*7t#4RPMqiAAhu*WUj+TMaJ=>~!KOzYBgB|lGPG8-aWtKYqvBD9(IA^F zE5CR#!{tnRJlRZs>ZsmWj>xuLXGucNwSv~1TMj0&S?c!Jci4lbr2I@@+%+QBCvJ&< zD)Jgf9Tag=nNNScMr9)Q$ka+bZ9C;455Hm;p)=qe6~5CHB@FL%wvke;(gS#o@X#Q) zh}<~vSayT0P4(k=O@+-~*HGj<=9@_=3}M*Gh+4cNZ?p4;mEiGxNTKdE(X5n2>jMF6 z3twL9O9}FmaLTJ9k$nJ~O4k7ArDean1?atj%ui9fJEfeqszUCtL5w7y2i>b27iFXN zA&(KQ@0CvziVU+ln^IX22G z)=u=Rd~wG({Iqt$De`%;Ud6ch)8hX6QuF8T{g!nDZ&U;H+gh}2EDCsCMHNbV;YJ8B zw{{COcrUi__0^H;h6Zo`wCQddu%)iUs;K}^PS(|O9wgTQSQ=b)1wrXjcjd~_cmMrH zPbl9kp~m@Pf8(5@e>`RWq374j&;FPFP{Of^mLhoTBpkfwLReDzqdmH(PDxGY(+8)UTROr}>U;(t=(hM*qFJ$xU=gTh>&CK{{Ca za+D~4bT57E(=dAP5+h$Q`GkRn-uji9(k~-c(-YopoK(rm{lkdPH0!?k%~|P^HxAM0(cK7XeRBvY%YL+j|!YisR9;R4jFdsh$t8F>cW+*45cXjl2IxNv)SAq-X zV+rha#3qDVjWZV7HApQ&WM%py*X`|G2BSEZ#dG%;TEo)3a=U;p18#jjWwyA)qR&bR%12%pdIb=#l15$wO|S^Ti@_q~a)c;8Ty z|AhLRPQ^vkiv{l=s43^~=eM^0xBPt(^kV(`2k7ZL}5B%293vwC-l^pohoxT}*QBePD=tdy&Z+QQa*MGC~s|5NszPhO( z@gMBGD2RSF^r9sCVW`hv0WS!ouXg_Y{1=tT4_E_|zuEa8U2+ll;u-P-_=xOpz!y&x zH6`?OkpuwXpML=7ttKP|0Kf + for { + archive <- IO(AddonArchive(dummyAddonUrl, "", "")) + path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir) + + aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some) + _ = { + assertEquals(aa.name, "dummy-addon") + assertEquals(aa.version, "2.9") + assertEquals(aa.url, dummyAddonUrl) + } + } yield () + } + + test("Read archive from zip file") { + for { + archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO]) + _ = { + assertEquals(archive.name, "dummy-addon") + assertEquals(archive.version, "2.9") + assertEquals(archive.url, dummyAddonUrl) + } + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala new file mode 100644 index 00000000..5ad59b14 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.logging.{Level, TestLoggingConfig} + +import munit._ + +class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + + override def docspellLogConfig = + super.docspellLogConfig.copy(minimumLevel = Level.Trace) + + tempDir.test("select docker if Dockerfile exists") { dir => + for { + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker)) + } yield () + } + + tempDir.test("select nix-flake if flake.nix exists") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.NixFlake)) + } yield () + } + + tempDir.test("select nix-flake and docker") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake)) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala new file mode 100644 index 00000000..9403d02c --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.common.Glob +import docspell.files.Zip +import docspell.logging.TestLoggingConfig + +import munit._ + +class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures { + val logger = docspell.logging.getLogger[IO] + + test("read meta from zip file") { + val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192)) + assertIO(meta, dummyAddonMeta) + } + + tempDir.test("read meta from directory") { dir => + for { + _ <- dummyAddonUrl + .readURL[IO](8192) + .through(Zip.unzip(8192, Glob.all)) + .through(Zip.saveTo(logger, dir, moveUp = true)) + .compile + .drain + meta <- AddonMeta.findInDirectory[IO](dir) + _ = assertEquals(meta, dummyAddonMeta) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala new file mode 100644 index 00000000..0a4bb08e --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.out.AddonOutput + +import io.circe.parser.decode +import munit.FunSuite + +class AddonOutputTest extends FunSuite { + + test("decode empty object") { + val out = decode[AddonOutput]("{}") + println(out) + } + + test("decode sample output") { + val jsonStr = + """{ "files": [ + | { + | "itemId": "qZDnyGIAJsXr", + | "textFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.txt" + | }, + | "pdfFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.pdf" + | } + | } + | ] + |} + |""".stripMargin + + val out = decode[AddonOutput](jsonStr) + println(out) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala new file mode 100644 index 00000000..1f361ae2 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala @@ -0,0 +1,98 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.util.concurrent.atomic.AtomicInteger + +import cats.effect.IO +import fs2.io.file.Path + +import docspell.common.LenientUri +import docspell.common.exec.Env +import docspell.logging.{Logger, TestLoggingConfig} + +import munit._ + +class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig { + + val logger = docspell.logging.getLogger[IO] + + val dummyContext = Context( + addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""), + meta = AddonMeta.empty("test", "1.0"), + baseDir = Path(""), + addonDir = Path(""), + outputDir = Path(""), + cacheDir = Path("") + ) + + test("firstSuccessful must stop on first success") { + val counter = new AtomicInteger(0) + val runner = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(runner, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("firstSuccessful must try with next on error") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.failWith[IO]("failed") + val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(fail, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("do not retry on decoding errors") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful(fail, increment, increment) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 0) + } yield () + } + + test("try on errors but stop on decoding error") { + val counter = new AtomicInteger(0) + val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val incrementFail = + new MockRunner(IO(counter.incrementAndGet()).void) + .as(AddonResult.executionFailed(new Exception("fail"))) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful( + incrementFail, + incrementFail, + decodeFail, + increment, + increment + ) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 2) + } yield () + } + + final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty) + extends AddonRunner[IO] { + val runnerType = Nil + def run( + logger: Logger[IO], + env: Env, + ctx: Context + ) = run.as(result) + + def as(r: AddonResult) = new MockRunner(run, r) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala new file mode 100644 index 00000000..c20ac112 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path, PosixPermissions} + +import docspell.addons.AddonExecutorConfig._ +import docspell.addons.AddonMeta._ +import docspell.addons.AddonTriggerType._ +import docspell.common.{Duration, LenientUri} +import docspell.logging.TestLoggingConfig + +import munit.CatsEffectSuite + +trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite => + + val files: Files[IO] = Files[IO] + + val dummyAddonUrl = + LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip")) + + val dummyAddonMeta = + AddonMeta( + meta = + AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some), + triggers = Some( + Set(Scheduled, FinalProcessItem, FinalReprocessItem) + ), + None, + runner = Runner( + nix = NixRunner(true).some, + docker = DockerRunner( + enable = true, + image = None, + build = "Dockerfile".some + ).some, + trivial = TrivialRunner(true, "src/addon.sh").some + ).some, + options = Options(networking = true, collectOutput = true).some + ) + + def baseTempDir: Path = + Path(s"/tmp/target/test-temp") + + val tempDir = + ResourceFixture[Path]( + Resource.eval(Files[IO].createDirectories(baseTempDir)) *> + Files[IO] + .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777")) + ) + + def testExecutorConfig( + runner: RunnerType, + runners: RunnerType* + ): AddonExecutorConfig = { + val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100)) + AddonExecutorConfig( + runner :: runners.toList, + Duration.minutes(2), + nspawn, + NixConfig("nix", Duration.minutes(2)), + DockerConfig("docker", Duration.minutes(2)) + ) + } +} diff --git a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala index 15ac3f24..9542d3bb 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala @@ -15,8 +15,8 @@ import fs2.io.file.{Files, Path} import docspell.analysis.classifier import docspell.analysis.classifier.TextClassifier._ import docspell.analysis.nlp.Properties -import docspell.common._ import docspell.common.syntax.FileSyntax._ +import docspell.common.util.File import docspell.logging.Logger import edu.stanford.nlp.classify.ColumnDataClassifier diff --git a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala index f8423492..28f9490b 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala @@ -14,6 +14,7 @@ import cats.implicits._ import docspell.analysis.NlpSettings import docspell.common._ +import docspell.common.util.File /** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and * initializing large objects. diff --git a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala index 86804836..570a8439 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala @@ -17,6 +17,7 @@ import fs2.io.file.Files import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ +import docspell.common.util.File import docspell.logging.TestLoggingConfig import munit._ diff --git a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala index d522ec6c..15bc0704 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala @@ -13,6 +13,7 @@ import cats.effect.unsafe.implicits.global import docspell.analysis.Env import docspell.common._ +import docspell.common.util.File import docspell.files.TestFiles import docspell.logging.TestLoggingConfig diff --git a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala index 015e771b..386e2a4a 100644 --- a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala +++ b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala @@ -20,6 +20,7 @@ trait AttachedEvent[R] { object AttachedEvent { + /** Only the result, no events. */ def only[R](v: R): AttachedEvent[R] = new AttachedEvent[R] { val value = v diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index a34eaf6a..cc55c72b 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -8,11 +8,14 @@ package docspell.backend import cats.effect._ +import docspell.backend.BackendCommands.EventContext import docspell.backend.auth.Login import docspell.backend.fulltext.CreateIndex import docspell.backend.ops._ import docspell.backend.signup.OSignup +import docspell.common.bc.BackendCommandRunner import docspell.ftsclient.FtsClient +import docspell.joexapi.client.JoexClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT import docspell.scheduler.JobStoreModule @@ -20,6 +23,7 @@ import docspell.store.Store import docspell.totp.Totp import emil.Emil +import org.http4s.client.Client trait BackendApp[F[_]] { @@ -35,6 +39,7 @@ trait BackendApp[F[_]] { def job: OJob[F] def item: OItem[F] def itemSearch: OItemSearch[F] + def attachment: OAttachment[F] def fulltext: OFulltext[F] def mail: OMail[F] def joex: OJoex[F] @@ -52,23 +57,30 @@ trait BackendApp[F[_]] { def fileRepository: OFileRepository[F] def itemLink: OItemLink[F] def downloadAll: ODownloadAll[F] + def addons: OAddons[F] + + def commands(eventContext: Option[EventContext]): BackendCommandRunner[F, Unit] } object BackendApp { def create[F[_]: Async]( + cfg: Config, store: Store[F], javaEmil: Emil[F], + httpClient: Client[F], ftsClient: FtsClient[F], pubSubT: PubSubT[F], schedulerModule: JobStoreModule[F], notificationMod: NotificationModule[F] ): Resource[F, BackendApp[F]] = for { + nodeImpl <- ONode(store) totpImpl <- OTotp(store, Totp.default) loginImpl <- Login[F](store, Totp.default) signupImpl <- OSignup[F](store) - joexImpl <- OJoex(pubSubT) + joexClient = JoexClient(httpClient) + joexImpl <- OJoex(pubSubT, nodeImpl, joexClient) collImpl <- OCollective[F]( store, schedulerModule.userTasks, @@ -80,7 +92,6 @@ object BackendApp { equipImpl <- OEquipment[F](store) orgImpl <- OOrganization(store) uploadImpl <- OUpload(store, schedulerModule.jobs) - nodeImpl <- ONode(store) jobImpl <- OJob(store, joexImpl, pubSubT) createIndex <- CreateIndex.resource(ftsClient, store) itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs) @@ -109,6 +120,16 @@ object BackendApp { fileRepoImpl <- OFileRepository(store, schedulerModule.jobs) itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl)) downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs)) + attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs)) + addonsImpl <- Resource.pure( + OAddons( + cfg.addons, + store, + schedulerModule.userTasks, + schedulerModule.jobs, + joexImpl + ) + ) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl @@ -139,5 +160,10 @@ object BackendApp { val fileRepository = fileRepoImpl val itemLink = itemLinkImpl val downloadAll = downloadAllImpl + val addons = addonsImpl + val attachment = attachImpl + + def commands(eventContext: Option[EventContext]) = + BackendCommands.fromBackend(this, eventContext) } } diff --git a/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala new file mode 100644 index 00000000..3550cfbf --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ + +import docspell.backend.BackendCommands.EventContext +import docspell.backend.ops.OCustomFields.SetValue +import docspell.backend.ops._ +import docspell.common.bc._ +import docspell.common.{AccountId, Ident, LenientUri} + +private[backend] class BackendCommands[F[_]: Sync]( + itemOps: OItem[F], + attachOps: OAttachment[F], + fieldOps: OCustomFields[F], + notificationOps: ONotification[F], + eventContext: Option[EventContext] +) extends BackendCommandRunner[F, Unit] { + private[this] val logger = docspell.logging.getLogger[F] + + def run(collective: Ident, cmd: BackendCommand): F[Unit] = + doRun(collective, cmd).attempt.flatMap { + case Right(_) => ().pure[F] + case Left(ex) => + logger.error(ex)(s"Backend command $cmd failed for collective ${collective.id}.") + } + + def doRun(collective: Ident, cmd: BackendCommand): F[Unit] = + cmd match { + case BackendCommand.ItemUpdate(item, actions) => + actions.traverse_(a => runItemAction(collective, item, a)) + + case BackendCommand.AttachmentUpdate(item, attach, actions) => + actions.traverse_(a => runAttachAction(collective, item, attach, a)) + } + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[Unit] = + cmds.traverse_(run(collective, _)) + + def runItemAction(collective: Ident, item: Ident, action: ItemAction): F[Unit] = + action match { + case ItemAction.AddTags(tags) => + logger.debug(s"Setting tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .linkTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.RemoveTags(tags) => + logger.debug(s"Remove tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .removeTagsMultipleItems(Nel.of(item), tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.ReplaceTags(tags) => + logger.debug(s"Replace tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .setTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.SetFolder(folder) => + logger.debug(s"Set folder $folder on ${item.id} for ${collective.id}") *> + itemOps + .setFolder(item, folder, collective) + .void + + case ItemAction.RemoveTagsCategory(cats) => + logger.debug( + s"Remove tags in categories $cats on ${item.id} for ${collective.id}" + ) *> + itemOps + .removeTagsOfCategories(item, collective, cats) + .flatMap(sendEvents) + + case ItemAction.SetCorrOrg(id) => + logger.debug( + s"Set correspondent organization ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrOrg(Nel.of(item), id, collective).void + + case ItemAction.SetCorrPerson(id) => + logger.debug( + s"Set correspondent person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcPerson(id) => + logger.debug( + s"Set concerning person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcEquipment(id) => + logger.debug( + s"Set concerning equipment ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcEquip(Nel.of(item), id, collective).void + + case ItemAction.SetField(field, value) => + logger.debug( + s"Set field on item ${item.id} ${field.id} to '$value' for ${collective.id}" + ) *> + fieldOps + .setValue(item, SetValue(field, value, collective)) + .flatMap(sendEvents) + + case ItemAction.SetNotes(notes) => + logger.debug(s"Set notes on item ${item.id} for ${collective.id}") *> + itemOps.setNotes(item, notes, collective).void + + case ItemAction.AddNotes(notes, sep) => + logger.debug(s"Add notes on item ${item.id} for ${collective.id}") *> + itemOps.addNotes(item, notes, sep, collective).void + + case ItemAction.SetName(name) => + logger.debug(s"Set name '$name' on item ${item.id} for ${collective.id}") *> + itemOps.setName(item, name, collective).void + } + + def runAttachAction( + collective: Ident, + itemId: Ident, + attachId: Ident, + action: AttachmentAction + ): F[Unit] = + action match { + case AttachmentAction.SetExtractedText(text) => + attachOps.setExtractedText( + collective, + itemId, + attachId, + text.getOrElse("").pure[F] + ) + } + + private def sendEvents(result: AttachedEvent[_]): F[Unit] = + eventContext match { + case Some(ctx) => + notificationOps.offerEvents(result.event(ctx.account, ctx.baseUrl)) + case None => ().pure[F] + } +} + +object BackendCommands { + + /** If supplied, notification events will be send. */ + case class EventContext(account: AccountId, baseUrl: Option[LenientUri]) + + def fromBackend[F[_]: Sync]( + backendApp: BackendApp[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F]( + backendApp.item, + backendApp.attachment, + backendApp.customFields, + backendApp.notification, + eventContext + ) + + def apply[F[_]: Sync]( + item: OItem[F], + attachment: OAttachment[F], + fields: OCustomFields[F], + notification: ONotification[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F](item, attachment, fields, notification, eventContext) +} diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index efccb8dc..b1baedcb 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -20,7 +20,8 @@ case class Config( mailDebug: Boolean, jdbc: JdbcConfig, signup: SignupConfig, - files: Config.Files + files: Config.Files, + addons: Config.Addons ) { def mailSettings: Settings = @@ -66,4 +67,21 @@ object Config { (storesEmpty |+| defaultStorePresent).map(_ => this) } } + + case class Addons( + enabled: Boolean, + allowImpure: Boolean, + allowedUrls: UrlMatcher, + deniedUrls: UrlMatcher + ) { + def isAllowed(url: LenientUri): Boolean = + allowedUrls.matches(url) && !deniedUrls.matches(url) + + def isDenied(url: LenientUri): Boolean = + !isAllowed(url) + } + object Addons { + val disabled: Addons = + Addons(false, false, UrlMatcher.False, UrlMatcher.True) + } } diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index 87adbbdc..cf87d783 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -16,6 +16,26 @@ import docspell.notification.api.PeriodicQueryArgs import docspell.scheduler.Job object JobFactory extends MailAddressCodec { + def existingItemAddon[F[_]: Sync]( + args: ItemAddonTaskArgs, + submitter: AccountId + ): F[Job[ItemAddonTaskArgs]] = + Job.createNew( + ItemAddonTaskArgs.taskName, + submitter.collective, + args, + "Run addons on item", + submitter.user, + Priority.High, + args.addonRunConfigs + .map(_.take(23)) + .toList + .sorted + .foldLeft(args.itemId)(_ / _) + .take(250) + .some + ) + def downloadZip[F[_]: Sync]( args: DownloadZipArgs, summaryId: Ident, diff --git a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala index a30e0922..fd5f9214 100644 --- a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala +++ b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala @@ -45,7 +45,14 @@ object CreateIndex { chunkSize: Int ): F[Unit] = { val attachs = store - .transact(QAttachment.allAttachmentMetaAndName(collective, itemIds, chunkSize)) + .transact( + QAttachment.allAttachmentMetaAndName( + collective, + itemIds, + ItemState.validStates, + chunkSize + ) + ) .map(caa => TextData .attachment( diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala new file mode 100644 index 00000000..dba1cbf4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import fs2.io.file.Path + +import docspell.addons.AddonExecutorConfig + +final case class AddonEnvConfig( + workingDir: Path, + cacheDir: Path, + executorConfig: AddonExecutorConfig +) diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala new file mode 100644 index 00000000..f69dd2a7 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala @@ -0,0 +1,199 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons._ +import docspell.backend.joex.AddonOps.{AddonRunConfigRef, ExecResult} +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.common.exec.Env +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records.AddonRunConfigResolved + +trait AddonOps[F[_]] { + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + middleware: Middleware[F] + ): F[ExecResult] + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + middleware: Middleware[F] + ): F[ExecResult] + + /** Find enabled addon run config references to be executed. Can be additionally + * filtered by given ids and triggers. + */ + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] + + /** Find enabled addon run config reference given an addon task id */ + def findAddonRef(collective: Ident, runConfigId: Ident): F[Option[AddonRunConfigRef]] + + /** Creates an executor for addons given a configuration. */ + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] + +} + +object AddonOps { + case class AddonRunConfigRef( + id: Ident, + collective: Ident, + userId: Option[Ident], + name: String, + refs: List[AddonRef] + ) + + object AddonRunConfigRef { + def fromResolved(r: AddonRunConfigResolved): AddonRunConfigRef = + AddonRunConfigRef( + r.config.id, + r.config.cid, + r.config.userId, + r.config.name, + r.refs.map(ref => AddonRef(ref.archive.asArchive, ref.ref.args)) + ) + } + + case class ExecResult( + result: List[AddonExecutionResult], + runConfigs: List[AddonRunConfigRef] + ) { + lazy val combined = result.combineAll + } + + object ExecResult { + def runConfigNotFound(id: Ident): ExecResult = + ExecResult( + AddonExecutionResult( + AddonResult.executionFailed( + new Exception(s"Addon run config ${id.id} not found.") + ) :: Nil, + false + ) :: Nil, + Nil + ) + } + + def apply[F[_]: Async]( + cfg: AddonEnvConfig, + store: Store[F], + cmdRunner: BackendCommandRunner[F, Unit], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonOps[F] = + new AddonOps[F] with LoggerExtension { + private[this] val logger = docspell.logging.getLogger[F] + + private val urlReader = FileUrlReader(store.fileRepo) + private val postProcess = AddonPostProcess(cmdRunner, store, attachment, jobStore) + private val prepare = new AddonPrepare[F](store) + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + custom: Middleware[F] + ): F[ExecResult] = + for { + runCfgs <- findAddonRefs(collective, trigger, runConfigIds) + log = logger.getOrElse(this.logger) + _ <- log.info(s"Running ${runCfgs.size} addon tasks for trigger $trigger") + + results <- runCfgs.traverse(r => execRunConfig(log, r, custom)) + } yield ExecResult(results.flatMap(_.result), runCfgs) + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + custom: Middleware[F] + ): F[ExecResult] = + (for { + runCfg <- OptionT(findAddonRef(collective, runConfigId)) + execRes <- OptionT.liftF(execRunConfig(logger, runCfg, custom)) + } yield execRes).getOrElse(ExecResult.runConfigNotFound(runConfigId)) + + def execRunConfig( + logger: Logger[F], + runCfg: AddonRunConfigRef, + custom: Middleware[F] + ): F[ExecResult] = + for { + executor <- getExecutor(cfg.executorConfig) + log = logger.withRunConfig(runCfg) + result <- + Directory.temp(cfg.workingDir, "addon-output-").use { outDir => + val cacheDir = cfg.cacheDir / runCfg.id.id + val inputEnv = + InputEnv(runCfg.refs, cfg.workingDir, outDir, cacheDir, Env.empty) + + for { + middleware <- createMiddleware(custom, runCfg) + res <- middleware(executor.execute(log)).run(inputEnv) + _ <- log.debug(s"Addon result: $res") + _ <- postProcess.onResult(log, runCfg.collective, res, outDir) + } yield res + } + execRes = ExecResult(List(result), List(runCfg)) + } yield execRes + + def createMiddleware(custom: Middleware[F], runCfg: AddonRunConfigRef) = for { + dscMW <- prepare.createDscEnv(runCfg, cfg.executorConfig.runTimeout) + mm = dscMW >> custom >> prepare.logResult(logger, runCfg) >> Middleware + .ephemeralRun[F] + } yield mm + + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] = + Async[F].pure(AddonExecutor(cfg, urlReader)) + + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] = + store + .transact( + AddonRunConfigResolved.findAllForCollective( + collective, + enabled = true.some, + trigger, + runConfigIds + ) + ) + .map(_.map(AddonRunConfigRef.fromResolved)) + + def findAddonRef( + collective: Ident, + runConfigId: Ident + ): F[Option[AddonRunConfigRef]] = + OptionT( + store + .transact( + AddonRunConfigResolved.findById( + runConfigId, + collective, + enabled = Some(true) + ) + ) + ).map(AddonRunConfigRef.fromResolved).value + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala new file mode 100644 index 00000000..2696b4e9 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala @@ -0,0 +1,198 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons._ +import docspell.addons.out.{AddonOutput, ItemFile, NewItem} +import docspell.backend.JobFactory +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.files.FileSupport +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.records._ + +final private[joex] class AddonPostProcess[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachOps: OAttachment[F], + jobStore: JobStore[F] +) extends FileSupport { + + def onResult( + logger: Logger[F], + collective: Ident, + result: AddonExecutionResult, + outputDir: Path + ): F[Unit] = + result.addonResult match { + case AddonResult.Success(output) => + onSuccess(logger, collective, output, outputDir) + case _ => + ().pure[F] + } + + def onSuccess( + logger: Logger[F], + collective: Ident, + output: AddonOutput, + outputDir: Path + ): F[Unit] = + for { + _ <- logger.info("Applying addon output") + _ <- cmdRunner.runAll(collective, output.commands) + _ <- logger.debug("Applying changes from files") + _ <- output.files.traverse_(updateOne(logger, collective, outputDir)) + _ <- output.newItems.traverse_(submitNewItem(logger, collective, outputDir)) + } yield () + + def submitNewItem( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(newItem: NewItem): F[Unit] = + for { + _ <- logger.info(s"Submit new item with ${newItem.files.size} files") + files <- newItem.resolveFiles[F](logger, outputDir) + collLang <- store.transact(RCollective.findLanguage(collective)) + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + _ <- logger.debug(s"Saved ${uploaded.size} files to be processed.") + args = ProcessItemArgs( + newItem.toProcessMeta(collective, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + account = AccountId(collective, DocspellSystem.user) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + + def updateOne(logger: Logger[F], collective: Ident, outputDir: Path)( + itemFile: ItemFile + ): F[Unit] = + for { + textFiles <- itemFile.resolveTextFiles(logger, outputDir) + pdfFiles <- itemFile.resolvePdfFiles(logger, outputDir) + previewFiles <- itemFile.resolvePreviewFiles(logger, outputDir) + attachs <- OptionT + .whenF(textFiles.nonEmpty || pdfFiles.nonEmpty || previewFiles.nonEmpty)( + store.transact(RAttachment.findByItem(itemFile.itemId)) + ) + .getOrElse(Vector.empty) + _ <- textFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + setText(collective, ra, file.readText) + } + } + _ <- pdfFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePdf(collective, ra, file, previewFiles.forall(_._1 != key)) + } + } + _ <- previewFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePreview(collective, ra.id, file) + } + } + _ <- submitNewFiles(logger, collective, outputDir)(itemFile) + } yield () + + def submitNewFiles( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(itemFile: ItemFile): F[Unit] = + for { + _ <- logger.info(s"Submitting new file for item") + collLang <- store.transact(RCollective.findLanguage(collective)) + newFiles <- itemFile.resolveNewFiles(logger, outputDir) + byMeta = newFiles.groupBy(_._1.metadata).view.mapValues(_.map(_._2)) + account = AccountId(collective, DocspellSystem.user) + _ <- byMeta.toList.traverse_ { case (meta, files) => + for { + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + args = ProcessItemArgs( + meta.toProcessMeta(collective, itemFile.itemId, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + } + } yield () + + private def withAttach(logger: Logger[F], key: String, attachs: Vector[RAttachment])( + run: RAttachment => F[Unit] + ): F[Unit] = + OptionT + .fromOption( + attachs.find(a => a.id.id == key || key.toIntOption == a.position.some) + ) + .semiflatMap(run) + .getOrElseF(logger.warn(s"Cannot find attachment for $key to update text!")) + + private def setText(collective: Ident, ra: RAttachment, readText: F[String]): F[Unit] = + attachOps.setExtractedText(collective, ra.itemId, ra.id, readText) + + private def replacePdf( + collective: Ident, + ra: RAttachment, + file: Path, + generatePreview: Boolean + ): F[Unit] = + attachOps.addOrReplacePdf(collective, ra.id, file.readAll, generatePreview) + + private def replacePreview( + collective: Ident, + attachId: Ident, + imageData: Path + ): F[Unit] = + attachOps.addOrReplacePreview(collective, attachId, imageData.readAll) +} + +object AddonPostProcess { + + def apply[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonPostProcess[F] = + new AddonPostProcess[F](cmdRunner, store, attachment, jobStore) +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala new file mode 100644 index 00000000..03ab223d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.auth.AuthToken +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.common._ +import docspell.logging.Logger +import docspell.store.Store +import docspell.store.records.{RNode, RUser} + +import scodec.bits.ByteVector + +private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExtension { + + def logResult(logger: Logger[F], ref: AddonRunConfigRef): Middleware[F] = + Middleware(_.mapF(_.attempt.flatTap { + case Right(_) => ().pure[F] + case Left(ex) => + logger + .withRunConfig(ref) + .warn(ex)(s"Addon task '${ref.id.id}' has failed") + }.rethrow)) + + /** Creates environment variables for dsc to connect to the docspell server for the + * given run config. + */ + def createDscEnv( + runConfigRef: AddonRunConfigRef, + tokenValidity: Duration + ): F[Middleware[F]] = + (for { + userId <- OptionT.fromOption[F](runConfigRef.userId) + user <- OptionT(store.transact(RUser.getIdByIdOrLogin(userId))) + account = AccountId(runConfigRef.collective, user.login) + env = + Middleware.prepare[F]( + Kleisli(input => makeDscEnv(account, tokenValidity).map(input.addEnv)) + ) + } yield env).getOrElse(Middleware.identity[F]) + + /** Creates environment variables to have dsc automatically connect as the given user. + * Additionally a random rest-server is looked up from the database to set its url. + */ + def makeDscEnv( + accountId: AccountId, + tokenValidity: Duration + ): F[Map[String, String]] = + for { + serverNode <- store.transact( + RNode + .findAll(NodeType.Restserver) + .map(_.sortBy(_.updated).lastOption) + ) + url = serverNode.map(_.url).map(u => "DSC_DOCSPELL_URL" -> u.asString) + secret = serverNode.flatMap(_.serverSecret) + + token <- AuthToken.user( + accountId, + false, + secret.getOrElse(ByteVector.empty), + tokenValidity.some + ) + session = ("DSC_SESSION" -> token.asString).some + } yield List(url, session).flatten.toMap +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala new file mode 100644 index 00000000..1f3e4da1 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.logging.Logger + +trait LoggerExtension { + + implicit final class LoggerDataOps[F[_]](self: Logger[F]) { + def withRunConfig(t: AddonRunConfigRef): Logger[F] = + self.capture("addon-task-id", t.id) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala new file mode 100644 index 00000000..e634233a --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} + +sealed trait AddonRunConfigError { + final def cast: AddonRunConfigError = this + + def toLeft[A]: Either[AddonRunConfigError, A] = Left(this) + + def message: String +} + +object AddonRunConfigError { + + case object MissingSchedule extends AddonRunConfigError { + val message = + "The run config has a trigger 'scheduled' but doesn't provide a schedule!" + } + + case object ObsoleteSchedule extends AddonRunConfigError { + val message = "The run config has a schedule, but not a trigger 'Scheduled'." + } + + case class MismatchingTrigger(unsupported: NonEmptyList[(String, AddonTriggerType)]) + extends AddonRunConfigError { + def message: String = { + val list = + unsupported.map { case (name, tt) => s"$name: ${tt.name}" }.toList.mkString(", ") + s"Some listed addons don't support all defined triggers: $list" + } + } + + object MismatchingTrigger { + def apply(addon: AddonMeta, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + + def apply(addon: AddonArchive, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala new file mode 100644 index 00000000..c675c13d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.ops.AddonRunConfigError._ +import docspell.backend.ops.OAddons.{AddonRunConfigResult, AddonRunInsert} +import docspell.common.Ident +import docspell.store.Store +import docspell.store.records.RAddonArchive + +object AddonRunConfigValidate { + + def apply[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[AddonRunInsert]] = { + val init: AddonRunConfigResult[Unit] = ().asRight + + List( + checkScheduled(cfg).pure[F], + checkTriggers(store, cid)(cfg) + ) + .foldLeftM(init)((res, fr) => fr.map(r => res.flatMap(_ => r))) + .map(_.as(cfg)) + } + + def checkTriggers[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[Unit]] = + for { + addons <- store.transact(RAddonArchive.findByIds(cid, cfg.addons.map(_.addonId))) + given = cfg.triggered.toList.toSet + res = addons + .flatMap(r => given.diff(r.triggers).map(tt => r.nameAndVersion -> tt)) + + maybeError = NonEmptyList + .fromList(res) + .map(nel => MismatchingTrigger(nel)) + } yield maybeError.map(_.toLeft).getOrElse(Right(())) + + def checkScheduled(cfg: AddonRunInsert): AddonRunConfigResult[Unit] = + (cfg.isScheduled, cfg.schedule) match { + case (true, None) => MissingSchedule.toLeft[Unit] + case (false, Some(_)) => ObsoleteSchedule.toLeft[Unit] + case _ => ().asRight + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala new file mode 100644 index 00000000..6074f557 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala @@ -0,0 +1,156 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.EitherT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.Path + +import docspell.addons.{AddonMeta, RunnerType} +import docspell.backend.Config +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.{Ident, LenientUri, UrlReader} +import docspell.joexapi.model.AddonSupport +import docspell.store.Store +import docspell.store.records.RAddonArchive + +final class AddonValidate[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + joexOps: OJoex[F] +) { + private[this] val logger = docspell.logging.getLogger[F] + + def fromUrl( + collective: Ident, + url: LenientUri, + reader: UrlReader[F], + localUrl: Option[LenientUri] = None, + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + if (!cfg.enabled) AddonsDisabled.resultF + else if (cfg.isDenied(url)) UrlUntrusted(url).resultF + else if (checkExisting) + store.transact(RAddonArchive.findByUrl(collective, url)).flatMap { + case Some(ar) => + AddonExists("An addon with this url already exists!", ar).resultF + case None => + archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + } + else archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + + def archive( + collective: Ident, + addonData: Either[Path, Stream[F, Byte]], + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + (for { + _ <- EitherT.cond[F](cfg.enabled, (), AddonsDisabled.cast) + + meta <- + EitherT( + addonData + .fold( + AddonMeta.findInDirectory[F], + AddonMeta.findInZip[F] + ) + .attempt + ) + .leftMap(ex => NotAnAddon(ex).cast) + _ <- EitherT.cond( + meta.triggers.exists(_.nonEmpty), + (), + InvalidAddon( + "The addon doesn't define any triggers. At least one is required!" + ).cast + ) + _ <- EitherT.cond( + meta.options.exists(_.isUseful), + (), + InvalidAddon( + "Addon defines no output and no networking. It can't do anything useful." + ).cast + ) + _ <- EitherT.cond(cfg.allowImpure || meta.isPure, (), ImpureAddonsDisabled.cast) + + _ <- + if (checkExisting) + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(collective, meta.meta.name, meta.meta.version) + ) + .map { + case Some(ar) => AddonExists(ar).result + case None => rightUnit + } + ) + else rightUnitT + + joexSupport <- EitherT.liftF(joexOps.getAddonSupport) + addonRunners <- EitherT.liftF(meta.enabledTypes(addonData)) + _ <- EitherT.liftF( + logger.info( + s"Comparing joex support vs addon runner: $joexSupport vs. $addonRunners" + ) + ) + _ <- EitherT.fromEither(validateJoexSupport(addonRunners, joexSupport)) + + } yield meta).value + + private def validateJoexSupport( + addonRunnerTypes: List[RunnerType], + joexSupport: List[AddonSupport] + ): AddonValidationResult[Unit] = { + val addonRunners = addonRunnerTypes.mkString(", ") + for { + _ <- Either.cond( + joexSupport.nonEmpty, + (), + AddonUnsupported("There are no joex nodes that have addons enabled!", Nil).cast + ) + _ <- Either.cond( + addonRunners.nonEmpty, + (), + InvalidAddon("The addon doesn't enable any runner.") + ) + + ids = joexSupport + .map(n => n.nodeId -> n.runners.intersect(addonRunnerTypes).toSet) + + unsupportedJoex = ids.filter(_._2.isEmpty).map(_._1) + + _ <- Either.cond( + ids.forall(_._2.nonEmpty), + (), + AddonUnsupported( + s"A joex node doesn't support this addons runners: $addonRunners. " + + s"Check: ${unsupportedJoex.map(_.id).mkString(", ")}.", + unsupportedJoex + ).cast + ) + } yield () + } + + private def rightUnit: AddonValidationResult[Unit] = + ().asRight[AddonValidationError] + + private def rightUnitT: EitherT[F, AddonValidationError, Unit] = + EitherT.fromEither(rightUnit) + + implicit final class ErrorOps(self: AddonValidationError) { + def result: AddonValidationResult[AddonMeta] = + self.toLeft + + def resultF: F[AddonValidationResult[AddonMeta]] = + result.pure[F] + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala new file mode 100644 index 00000000..d8f77e24 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import docspell.common.{Ident, LenientUri} +import docspell.store.records.RAddonArchive + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AddonValidationError { + def cast: AddonValidationError = this + + def toLeft[A]: Either[AddonValidationError, A] = Left(this) +} + +object AddonValidationError { + + implicit private val throwableDecoder: Decoder[Throwable] = + Decoder.decodeString.map(new Exception(_)) + implicit private val throwableEncoder: Encoder[Throwable] = + Encoder.encodeString.contramap(_.getMessage) + + case object AddonsDisabled extends AddonValidationError {} + + case class UrlUntrusted(url: LenientUri) extends AddonValidationError + object UrlUntrusted { + implicit val jsonDecoder: Decoder[UrlUntrusted] = deriveDecoder + implicit val jsonEncoder: Encoder[UrlUntrusted] = deriveEncoder + } + + case class NotAnAddon(error: Throwable) extends AddonValidationError + object NotAnAddon { + implicit val jsonDecoder: Decoder[NotAnAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[NotAnAddon] = deriveEncoder + } + + case class AddonUnsupported(message: String, affectedNodes: List[Ident]) + extends AddonValidationError + object AddonUnsupported { + implicit val jsonDecoder: Decoder[AddonUnsupported] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonUnsupported] = deriveEncoder + } + + case class InvalidAddon(message: String) extends AddonValidationError + object InvalidAddon { + implicit val jsonDecoder: Decoder[InvalidAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[InvalidAddon] = deriveEncoder + } + + case class AddonExists(message: String, addon: RAddonArchive) + extends AddonValidationError + object AddonExists { + def apply(addon: RAddonArchive): AddonExists = + AddonExists(s"An addon '${addon.name}/${addon.version}' already exists!", addon) + + implicit val jsonDecoder: Decoder[AddonExists] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonExists] = deriveEncoder + } + + case object AddonNotFound extends AddonValidationError + + case class DownloadFailed(error: Throwable) extends AddonValidationError + object DownloadFailed { + implicit val jsonDecoder: Decoder[DownloadFailed] = deriveDecoder + implicit val jsonEncoder: Encoder[DownloadFailed] = deriveEncoder + } + + case object ImpureAddonsDisabled extends AddonValidationError + + case object RefreshLocalAddon extends AddonValidationError + + implicit val jsonConfig: Configuration = + Configuration.default.withKebabCaseConstructorNames + .withDiscriminator("errorType") + + implicit val jsonDecoder: Decoder[AddonValidationError] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonValidationError] = deriveConfiguredEncoder +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala new file mode 100644 index 00000000..be5232a4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala @@ -0,0 +1,426 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{EitherT, NonEmptyList, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.{AddonMeta, AddonTriggerType} +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons._ +import docspell.backend.{Config, JobFactory} +import docspell.common._ +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records._ + +import com.github.eikek.calev.CalEvent + +trait OAddons[F[_]] { + + /** Registers a new addon. An error is returned if an addon with this url already + * exists. + */ + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Refreshes an existing addon by downloading it again and updating metadata. */ + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Look into the addon at the given url and return its metadata. */ + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] + + /** Deletes the addon if it exists. */ + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] + + /** Inserts or updates the addon run configuration. If it already exists (and the given + * id is non empty), it will be completely replaced with the given one. + */ + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] + + /** Deletes this task from the database. */ + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] +} + +object OAddons { + val scheduledAddonTaskName: Ident = + ScheduledAddonTaskArgs.taskName + + case class AddonRunInsert( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: NonEmptyList[AddonTriggerType], + addons: NonEmptyList[AddonArgs] + ) { + + def isScheduled: Boolean = + triggered.exists(_ == AddonTriggerType.Scheduled) + } + case class AddonArgs(addonId: Ident, args: String) + + case class AddonRunInfo( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: List[AddonTriggerType], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + ) + object AddonRunInfo { + def fromRunConfigData( + timer: Option[CalEvent], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + )(t: AddonRunConfigData): AddonRunInfo = + AddonRunInfo( + id = t.runConfig.id, + name = t.runConfig.name, + enabled = t.runConfig.enabled, + userId = t.runConfig.userId, + schedule = timer, + triggered = t.triggers.map(_.trigger), + addons = addons + ) + } + + type AddonRunConfigResult[A] = Either[AddonRunConfigError, A] + object AddonRunConfigResult { + def success[A](value: A): AddonRunConfigResult[A] = Right(value) + def failure[A](error: AddonRunConfigError): AddonRunConfigResult[A] = error.toLeft[A] + } + + type AddonValidationResult[A] = Either[AddonValidationError, A] + object AddonValidationResult { + def success[A](value: A): AddonValidationResult[A] = Right(value) + def failure[A](error: AddonValidationError): AddonValidationResult[A] = Left(error) + } + + def apply[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + userTasks: UserTaskStore[F], + jobStore: JobStore[F], + joex: OJoex[F] + ): OAddons[F] = + new OAddons[F] { + private[this] val logger = docspell.logging.getLogger[F] + private val urlReader = FileUrlReader(store.fileRepo) + private val zip = MimeType.zip.asString + private val addonValidate = new AddonValidate[F](cfg, store, joex) + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] = + for { + all <- store.transact(AddonRunConfigData.findAll(collective)) + runConfigIDs = all.map(_.runConfig.id).toSet + archiveIds = all.flatMap(_.addons.map(_.addonId)).distinct + archives <- NonEmptyList + .fromList(archiveIds) + .fold(List.empty[RAddonArchive].pure[F])(ids => + store.transact(RAddonArchive.findByIds(collective, ids)) + ) + archivesMap = archives.groupBy(_.id) + ptask <- userTasks + .getAll(UserTaskScope.collective(collective)) + .filter(ut => runConfigIDs.contains(ut.id)) + .map(ut => ut.id -> ut) + .compile + .toList + .map(_.toMap) + result = all.map { t => + AddonRunInfo.fromRunConfigData( + ptask.get(t.runConfig.id).map(_.timer), + t.addons.map(raa => (archivesMap(raa.addonId).head, raa)) + )(t) + } + } yield result + + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] = { + val insertDataRaw = AddonRunConfigData( + RAddonRunConfig( + runConfig.id, + collective, + runConfig.userId, + runConfig.name, + runConfig.enabled, + Timestamp.Epoch + ), + runConfig.addons.zipWithIndex.map { case (a, index) => + RAddonRunConfigAddon(Ident.unsafe(""), runConfig.id, a.addonId, a.args, index) + }.toList, + runConfig.triggered + .map(t => RAddonRunConfigTrigger(Ident.unsafe(""), runConfig.id, t)) + .toList + ) + + val upsert = for { + userId <- + OptionT + .fromOption(runConfig.userId) + .flatMapF(uid => store.transact(RUser.getIdByIdOrLogin(uid))) + .map(_.uid) + .value + insertData = + insertDataRaw.copy(runConfig = + insertDataRaw.runConfig.copy(userId = userId.orElse(runConfig.userId)) + ) + id <- + OptionT(store.transact(RAddonRunConfig.findById(collective, runConfig.id))) + .map(rt => + AddonRunConfigData( + rt.copy( + userId = insertData.runConfig.userId, + name = insertData.runConfig.name, + enabled = insertData.runConfig.enabled + ), + insertData.addons, + insertData.triggers + ) + ) + .semiflatMap(rt => + store.transact(AddonRunConfigData.update(rt).as(rt.runConfig.id)) + ) + .getOrElseF(store.transact(AddonRunConfigData.insert(insertData))) + } yield id + + EitherT(AddonRunConfigValidate(store, collective)(runConfig)) + .semiflatMap(_ => + upsert.flatTap { runConfigId => + runConfig.schedule match { + case Some(timer) => + userTasks.updateTask( + UserTaskScope.collective(collective), + s"Addon task ${runConfig.name}".some, + UserTask( + runConfigId, + scheduledAddonTaskName, + true, + timer, + s"Running scheduled addon task ${runConfig.name}".some, + ScheduledAddonTaskArgs(collective, runConfigId) + ) + ) + case None => + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + } + } + ) + .value + } + + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] = { + val deleteRunConfig = + (for { + e <- OptionT(RAddonRunConfig.findById(collective, runConfigId)) + _ <- OptionT.liftF(RAddonRunConfigAddon.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfigTrigger.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfig.deleteById(collective, e.id)) + } yield true).getOrElse(false) + + for { + deleted <- store.transact(deleteRunConfig) + _ <- + if (deleted) + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + else 0.pure[F] + } yield deleted + } + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] = + store.transact(RAddonArchive.listAll(collective)) + + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] = + store.transact(RAddonArchive.deleteById(collective, addonId)).map(_ > 0) + + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] = + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val log = logger.getOrElse(this.logger) + def validateAndInsert(file: FileKey, localUrl: LenientUri) = + addonValidate.fromUrl(collective, url, urlReader, localUrl.some).flatMap { + case Right(meta) => + insertAddon(collective, url, meta, file) + .map(ar => AddonValidationResult.success(ar -> meta)) + + case Left(error) => + store.fileRepo + .delete(file) + .as(AddonValidationResult.failure[(RAddonArchive, AddonMeta)](error)) + } + + log.info(s"Store addon file from '${url.asString} for ${collective.id}") *> + storeAddonFromUrl(collective, url).flatMapF { file => + val localUrl = FileUrlReader.url(file) + for { + _ <- log.info(s"Validating addon…") + res <- validateAndInsert(file, localUrl) + _ <- log.info(s"Validation result: $res") + } yield res + }.value + } + + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val findAddon = store + .transact(RAddonArchive.findById(collective, addonId)) + .map(_.toRight(AddonNotFound)) + def validateAddon(aa: RAddonArchive): F[AddonValidationResult[AddonMeta]] = + aa.originalUrl.fold( + AddonValidationResult.failure[AddonMeta](RefreshLocalAddon).pure[F] + )(url => + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + ) + + EitherT(findAddon).flatMap { aa => + EitherT(validateAddon(aa)) + .flatMap(meta => refreshAddon(aa, meta).map(na => na -> meta)) + }.value + } + + private def refreshAddon( + r: RAddonArchive, + meta: AddonMeta + ): EitherT[F, AddonValidationError, RAddonArchive] = + if (r.isUnchanged(meta)) EitherT.pure(r) + else + r.originalUrl match { + case Some(url) => + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(r.cid, meta.meta.name, meta.meta.version) + ) + .map( + _.fold(().asRight[AddonValidationError])(rx => AddonExists(rx).toLeft) + ) + ).flatMap(_ => + storeAddonFromUrl(r.cid, url).flatMap { file => + val nr = r.update(file, meta) + for { + _ <- EitherT( + store + .transact(RAddonArchive.update(nr)) + .map(_.asRight[AddonValidationError]) + .recoverWith { case ex => + logger.warn(ex)(s"Storing addon metadata failed.") *> + store.fileRepo + .delete(file) + .as( + AddonExists( + s"The addon '${nr.name}/${nr.version}' could not be stored", + nr + ).toLeft + ) + } + ) + _ <- EitherT.liftF(store.fileRepo.delete(r.fileId)) + } yield nr + } + ) + case None => + EitherT.leftT(RefreshLocalAddon.cast) + } + + private def insertAddon( + collective: Ident, + url: LenientUri, + meta: AddonMeta, + file: FileKey + ): F[RAddonArchive] = + for { + now <- Timestamp.current[F] + aId <- Ident.randomId[F] + record = RAddonArchive( + aId, + collective, + file, + url.some, + meta, + now + ) + _ <- store + .transact(RAddonArchive.insert(record, silent = false)) + .onError(_ => store.fileRepo.delete(file)) + } yield record + + private def storeAddonFromUrl(collective: Ident, url: LenientUri) = + for { + urlFile <- EitherT.pure(url.path.segments.lastOption) + file <- EitherT( + urlReader(url) + .through( + store.fileRepo.save( + collective, + FileCategory.Addon, + MimeTypeHint(urlFile, zip.some) + ) + ) + .compile + .lastOrError + .attempt + .map(_.leftMap(DownloadFailed(_).cast)) + ) + } yield file + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] = + for { + jobs <- itemIds.traverse(id => + JobFactory.existingItemAddon( + ItemAddonTaskArgs(account.collective, id, addonRunConfigIds), + account + ) + ) + _ <- jobStore.insertAllIfNew(jobs.map(_.encode).toList) + } yield () + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala new file mode 100644 index 00000000..b5f0ddd8 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala @@ -0,0 +1,223 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{NonEmptyList => Nel, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.backend.JobFactory +import docspell.common.MakePreviewArgs.StoreMode +import docspell.common._ +import docspell.files.TikaMimetype +import docspell.ftsclient.{FtsClient, TextData} +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +trait OAttachment[F[_]] { + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] +} + +object OAttachment { + + def apply[F[_]: Sync]( + store: Store[F], + fts: FtsClient[F], + jobStore: JobStore[F] + ): OAttachment[F] = + new OAttachment[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] = + for { + _ <- logger.info(s"Find attachment ${attachId.id} to update extracted text.") + cca <- store + .transact( + QAttachment + .allAttachmentMetaAndName( + collective.some, + Nel.of(itemId).some, + ItemState.validStates.append(ItemState.Processing), + 100 + ) + ) + .filter(_.id == attachId) + .compile + .last + content = cca.find(_.id == attachId) + _ <- logger.debug(s"Found existing metadata: ${content.isDefined}") + _ <- OptionT + .fromOption(content) + .semiflatMap { cnt => + for { + _ <- logger.debug(s"Setting new extracted text on ${cnt.id.id}") + text <- newText + td = TextData.attachment( + cnt.item, + cnt.id, + cnt.collective, + cnt.folder, + cnt.lang, + cnt.name, + text.some + ) + _ <- store.transact(RAttachmentMeta.updateContent(attachId, text)) + _ <- fts.updateIndex(logger, td) + } yield () + } + .getOrElseF( + logger.warn( + s"Item or attachment meta not found to update text: ${itemId.id}" + ) + ) + } yield () + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] = { + def generatePreview(ra: RAttachment): F[Unit] = + JobFactory + .makePreview(MakePreviewArgs(ra.id, StoreMode.Replace), None) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to re-generate preview from new pdf") + + def generatePageCount(ra: RAttachment): F[Unit] = + JobFactory + .makePageCount( + MakePageCountArgs(ra.id), + AccountId(collective, DocspellSystem.user).some + ) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to find page count from new pdf") + + def setFile(ra: RAttachment, rs: RAttachmentSource) = + for { + _ <- requireMimeType(pdfData, MimeType.pdf) + + newFile <- pdfData + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentConvert, + MimeTypeHint.advertised(MimeType.pdf) + ) + ) + .compile + .lastOrError + + _ <- store.transact(RAttachment.updateFileId(attachId, newFile)) + _ <- logger.info(s"Deleting old file for attachment") + _ <- + if (rs.fileId == ra.fileId) ().pure[F] + else store.fileRepo.delete(ra.fileId) + _ <- + if (regeneratePreview) generatePreview(ra) + else ().pure[F] + _ <- generatePageCount(ra) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + rs <- OptionT( + store.transact(RAttachmentSource.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra, rs)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot replace pdf file. Attachment not found for id: ${attachId.id}" + ) + ) + } + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] = { + def setFile(ra: RAttachment): F[Unit] = + for { + _ <- requireMimeType(imageData, MimeType.image("*")) + newFile <- imageData + .through( + store.fileRepo + .save(collective, FileCategory.PreviewImage, MimeTypeHint.none) + ) + .compile + .lastOrError + + now <- Timestamp.current[F] + record = RAttachmentPreview(ra.id, newFile, None, now) + oldFile <- store.transact(RAttachmentPreview.upsert(record)) + _ <- OptionT + .fromOption(oldFile) + .semiflatMap(store.fileRepo.delete) + .getOrElse(()) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot add/replace preview file. Attachment not found for id: ${attachId.id}" + ) + ) + } + } + + private def requireMimeType[F[_]: Sync]( + data: Stream[F, Byte], + expectedMime: MimeType + ): F[Unit] = + TikaMimetype + .detect(data, MimeTypeHint.advertised(expectedMime)) + .flatMap { mime => + if (expectedMime.matches(mime)) ().pure[F] + else + Sync[F].raiseError( + new IllegalArgumentException( + s"Expected pdf file, but got: ${mime.asString}" + ) + ) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 391a6593..79b799c0 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -61,6 +61,12 @@ trait OItem[F[_]] { collective: Ident ): F[AttachedEvent[UpdateResult]] + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -80,11 +86,13 @@ trait OItem[F[_]] { collective: Ident ): F[UpdateResult] - def setFolder(item: Ident, folder: Option[Ident], collective: Ident): F[UpdateResult] + /** Set or remove the folder on an item. Folder can be the id or name. */ + def setFolder(item: Ident, folder: Option[String], collective: Ident): F[UpdateResult] + /** Set or remove the folder on multiple items. Folder can be the id or name. */ def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] @@ -122,6 +130,13 @@ trait OItem[F[_]] { def setNotes(item: Ident, notes: Option[String], collective: Ident): F[UpdateResult] + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] def setNameMultiple( @@ -288,6 +303,28 @@ object OItem { } } + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] = + if (categories.isEmpty) { + AttachedEvent.only(UpdateResult.success).pure[F] + } else { + val dbtask = + for { + tags <- RTag.findByItem(item) + removeTags = tags.filter(_.category.exists(categories.contains)) + _ <- RTagItem.removeAllTags(item, removeTags.map(_.tagId)) + mkEvent = Event.TagsChanged + .partial(Nel.of(item), Nil, removeTags.map(_.tagId.id).toList) + } yield AttachedEvent(UpdateResult.success)(mkEvent) + + OptionT(store.transact(RItem.checkByIdAndCollective(item, collective))) + .semiflatMap(_ => store.transact(dbtask)) + .getOrElse(AttachedEvent.only(UpdateResult.notFound)) + } + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -420,21 +457,27 @@ object OItem { def setFolder( item: Ident, - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = - UpdateResult - .fromUpdate( - store - .transact(RItem.updateFolder(item, collective, folder)) + for { + result <- store.transact(RItem.updateFolder(item, collective, folder)).attempt + ures = result.fold( + UpdateResult.failure, + t => UpdateResult.fromUpdateRows(t._1) ) - .flatTap( - onSuccessIgnoreError(fts.updateFolder(logger, item, collective, folder)) + _ <- result.fold( + _ => ().pure[F], + t => + onSuccessIgnoreError(fts.updateFolder(logger, item, collective, t._2))( + ures + ) ) + } yield ures def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = for { @@ -615,6 +658,33 @@ object OItem { } ) + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] = + store + .transact(RItem.appendNotes(item, collective, notes, separator)) + .flatMap { + case Some(newNotes) => + store + .transact(RCollective.findLanguage(collective)) + .map(_.getOrElse(Language.English)) + .flatMap(lang => + fts.updateItemNotes(logger, item, collective, lang, newNotes.some) + ) + .attempt + .flatMap { + case Right(()) => ().pure[F] + case Left(ex) => + logger.warn(s"Error updating full-text index: ${ex.getMessage}") + } + .as(UpdateResult.success) + case None => + UpdateResult.notFound.pure[F] + } + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] = UpdateResult .fromUpdate( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index 9f83d46c..bc0b480c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -6,11 +6,13 @@ package docspell.backend.ops -import cats.Applicative import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import fs2.Stream -import docspell.common.Ident +import docspell.common.{Ident, NodeType} +import docspell.joexapi.client.JoexClient +import docspell.joexapi.model.AddonSupport import docspell.pubsub.api.PubSubT import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify} @@ -21,10 +23,16 @@ trait OJoex[F[_]] { def notifyPeriodicTasks: F[Unit] def cancelJob(job: Ident, worker: Ident): F[Unit] + + def getAddonSupport: F[List[AddonSupport]] } object OJoex { - def apply[F[_]: Applicative](pubSub: PubSubT[F]): Resource[F, OJoex[F]] = + def apply[F[_]: Async]( + pubSub: PubSubT[F], + nodes: ONode[F], + joexClient: JoexClient[F] + ): Resource[F, OJoex[F]] = Resource.pure[F, OJoex[F]](new OJoex[F] { def notifyAllNodes: F[Unit] = @@ -35,5 +43,17 @@ object OJoex { def cancelJob(job: Ident, worker: Ident): F[Unit] = pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(()) + + def getAddonSupport: F[List[AddonSupport]] = + for { + joex <- nodes.getNodes(NodeType.Joex) + conc = math.max(2, Runtime.getRuntime.availableProcessors() - 1) + supp <- Stream + .emits(joex) + .covary[F] + .parEvalMap(conc)(n => joexClient.getAddonSupport(n.url)) + .compile + .toList + } yield supp }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala index 8b55ed29..2e729c05 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala @@ -13,11 +13,27 @@ import docspell.common.{Ident, LenientUri, NodeType} import docspell.store.Store import docspell.store.records.RNode +import scodec.bits.ByteVector + trait ONode[F[_]] { - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] def unregister(appId: Ident): F[Unit] + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] + + def getNodes(nodeType: NodeType): F[Vector[RNode]] } object ONode { @@ -25,9 +41,14 @@ object ONode { def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] = Resource.pure[F, ONode[F]](new ONode[F] { val logger = docspell.logging.getLogger[F] - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] = for { - node <- RNode(appId, nodeType, uri) + node <- RNode(appId, nodeType, uri, serverSecret) _ <- logger.info(s"Registering node ${node.id.id}") _ <- store.transact(RNode.set(node)) } yield () @@ -35,6 +56,19 @@ object ONode { def unregister(appId: Ident): F[Unit] = logger.info(s"Unregister app ${appId.id}") *> store.transact(RNode.delete(appId)).map(_ => ()) + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] = + Resource.make(register(appId, nodeType, uri, serverSecret))(_ => + unregister(appId) + ) + + def getNodes(nodeType: NodeType): F[Vector[RNode]] = + store.transact(RNode.findAll(nodeType)) }) } diff --git a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala index 00453680..fdf77e2b 100644 --- a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala +++ b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala @@ -9,8 +9,9 @@ package docspell.common import java.time.Instant import io.circe._ +import scodec.bits.ByteVector -object BaseJsonCodecs { +trait BaseJsonCodecs { implicit val encodeInstantEpoch: Encoder[Instant] = Encoder.encodeJavaLong.contramap(_.toEpochMilli) @@ -18,4 +19,11 @@ object BaseJsonCodecs { implicit val decodeInstantEpoch: Decoder[Instant] = Decoder.decodeLong.map(Instant.ofEpochMilli) + implicit val byteVectorEncoder: Encoder[ByteVector] = + Encoder.encodeString.contramap(_.toBase64) + + implicit val byteVectorDecoder: Decoder[ByteVector] = + Decoder.decodeString.emap(ByteVector.fromBase64Descriptive(_)) } + +object BaseJsonCodecs extends BaseJsonCodecs diff --git a/modules/common/src/main/scala/docspell/common/Binary.scala b/modules/common/src/main/scala/docspell/common/Binary.scala index 2dc3c354..fa36ed1c 100644 --- a/modules/common/src/main/scala/docspell/common/Binary.scala +++ b/modules/common/src/main/scala/docspell/common/Binary.scala @@ -18,6 +18,18 @@ final case class Binary[F[_]](name: String, mime: MimeType, data: Stream[F, Byte def withMime(mime: MimeType): Binary[F] = copy(mime = mime) + + /** Return the extension of `name` if available (without the dot) */ + def extension: Option[String] = + name.lastIndexOf('.') match { + case n if n > 0 => + Some(name.substring(n + 1)) + case _ => + None + } + + def extensionIn(extensions: Set[String]): Boolean = + extension.exists(extensions.contains) } object Binary { diff --git a/modules/common/src/main/scala/docspell/common/FileCategory.scala b/modules/common/src/main/scala/docspell/common/FileCategory.scala index cdf6c723..21caa25c 100644 --- a/modules/common/src/main/scala/docspell/common/FileCategory.scala +++ b/modules/common/src/main/scala/docspell/common/FileCategory.scala @@ -32,6 +32,7 @@ object FileCategory { case object PreviewImage extends FileCategory case object Classifier extends FileCategory case object DownloadAll extends FileCategory + case object Addon extends FileCategory val all: NonEmptyList[FileCategory] = NonEmptyList.of( @@ -39,7 +40,8 @@ object FileCategory { AttachmentConvert, PreviewImage, Classifier, - DownloadAll + DownloadAll, + Addon ) def fromString(str: String): Either[String, FileCategory] = diff --git a/modules/common/src/main/scala/docspell/common/Glob.scala b/modules/common/src/main/scala/docspell/common/Glob.scala index 722af9fd..5b7ee9ca 100644 --- a/modules/common/src/main/scala/docspell/common/Glob.scala +++ b/modules/common/src/main/scala/docspell/common/Glob.scala @@ -32,7 +32,8 @@ object Glob { def single(str: String) = PatternGlob(Pattern(split(str, separator).map(makeSegment))) - if (in == "*") all + if (in == all.asString) all + else if (in == none.asString) none else split(in, anyChar) match { case NonEmptyList(_, Nil) => @@ -51,15 +52,25 @@ object Glob { val asString = "*" } + val none = new Glob { + def matches(caseSensitive: Boolean)(in: String) = false + def matchFilenameOrPath(in: String) = false + def asString = "!*" + } + def pattern(pattern: Pattern): Glob = PatternGlob(pattern) /** A simple glob supporting `*` and `?`. */ final private case class PatternGlob(pattern: Pattern) extends Glob { - def matches(caseSensitive: Boolean)(in: String): Boolean = + def matches(caseSensitive: Boolean)(in: String): Boolean = { + val input = Glob.split(in, Glob.separator) + + pattern.parts.size == input.size && pattern.parts - .zipWith(Glob.split(in, Glob.separator))(_.matches(caseSensitive)(_)) + .zipWith(input)(_.matches(caseSensitive)(_)) .forall(identity) + } def matchFilenameOrPath(in: String): Boolean = if (pattern.parts.tail.isEmpty) matches(true)(split(in, separator).last) @@ -67,6 +78,8 @@ object Glob { def asString: String = pattern.asString + + override def toString = s"PatternGlob($asString)" } final private case class AnyGlob(globs: NonEmptyList[Glob]) extends Glob { @@ -76,6 +89,8 @@ object Glob { globs.exists(_.matchFilenameOrPath(in)) def asString = globs.toList.map(_.asString).mkString(anyChar.toString) + + override def toString = s"AnyGlob($globs)" } case class Pattern(parts: NonEmptyList[Segment]) { diff --git a/modules/common/src/main/scala/docspell/common/Ident.scala b/modules/common/src/main/scala/docspell/common/Ident.scala index 2f630e33..928f0dd5 100644 --- a/modules/common/src/main/scala/docspell/common/Ident.scala +++ b/modules/common/src/main/scala/docspell/common/Ident.scala @@ -26,6 +26,9 @@ case class Ident(id: String) { def /(next: Ident): Ident = new Ident(id + Ident.concatChar + next.id) + + def take(n: Int): Ident = + new Ident(id.take(n)) } object Ident { diff --git a/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala new file mode 100644 index 00000000..3333486d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +/** Arguments to submit a task that runs addons configured for some existing item. + * + * If `addonTaskIds` is non empty, only these addon tasks are run. Otherwise all addon + * tasks that are configured for 'existing-item' are run. + */ +final case class ItemAddonTaskArgs( + collective: Ident, + itemId: Ident, + addonRunConfigs: Set[Ident] +) + +object ItemAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-existing-item") + + implicit val jsonDecoder: Decoder[ItemAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala index 55fbbda1..72da5281 100644 --- a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala +++ b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala @@ -6,6 +6,8 @@ package docspell.common +import fs2.io.file.Path + case class MimeTypeHint(filename: Option[String], advertised: Option[String]) { def withName(name: String): MimeTypeHint = @@ -21,6 +23,9 @@ object MimeTypeHint { def filename(name: String): MimeTypeHint = MimeTypeHint(Some(name), None) + def filename(file: Path): MimeTypeHint = + filename(file.fileName.toString) + def advertised(mimeType: MimeType): MimeTypeHint = advertised(mimeType.asString) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index 8398eaf0..9a92cee7 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -17,7 +17,7 @@ import io.circe.generic.semiauto._ * This task is run for each new file to create a new item from it or to add this file as * an attachment to an existing item. * - * If the `itemId' is set to some value, the item is tried to load to ammend with the + * If the `itemId' is set to some value, the item is tried to load to amend with the * given files. Otherwise a new item is created. * * It is also re-used by the 'ReProcessItem' task. diff --git a/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala new file mode 100644 index 00000000..5104108e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class ScheduledAddonTaskArgs(collective: Ident, addonTaskId: Ident) + +object ScheduledAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-scheduled-task") + + implicit val jsonDecoder: Decoder[ScheduledAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ScheduledAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/SystemCommand.scala b/modules/common/src/main/scala/docspell/common/SystemCommand.scala index 63f4ca40..702c546d 100644 --- a/modules/common/src/main/scala/docspell/common/SystemCommand.scala +++ b/modules/common/src/main/scala/docspell/common/SystemCommand.scala @@ -17,11 +17,23 @@ import cats.implicits._ import fs2.io.file.Path import fs2.{Stream, io, text} +import docspell.common.{exec => newExec} import docspell.logging.Logger +// better use `SysCmd` and `SysExec` object SystemCommand { - final case class Config(program: String, args: Seq[String], timeout: Duration) { + final case class Config( + program: String, + args: Seq[String], + timeout: Duration, + env: Map[String, String] = Map.empty + ) { + + def toSysCmd = newExec + .SysCmd(program, newExec.Args(args)) + .withTimeout(timeout) + .addEnv(newExec.Env(env)) def mapArgs(f: String => String): Config = Config(program, args.map(f), timeout) @@ -33,6 +45,18 @@ object SystemCommand { } ) + def withEnv(key: String, value: String): Config = + copy(env = env.updated(key, value)) + + def addEnv(moreEnv: Map[String, String]): Config = + copy(env = env ++ moreEnv) + + def appendArgs(extraArgs: Args): Config = + copy(args = args ++ extraArgs.args) + + def appendArgs(extraArgs: Seq[String]): Config = + copy(args = args ++ extraArgs) + def toCmd: List[String] = program :: args.toList @@ -40,6 +64,45 @@ object SystemCommand { toCmd.mkString(" ") } + final case class Args(args: Vector[String]) extends Iterable[String] { + override def iterator = args.iterator + + def prepend(a: String): Args = Args(a +: args) + + def prependWhen(flag: Boolean)(a: String): Args = + prependOption(Option.when(flag)(a)) + + def prependOption(value: Option[String]): Args = + value.map(prepend).getOrElse(this) + + def append(a: String, as: String*): Args = + Args(args ++ (a +: as.toVector)) + + def appendOption(value: Option[String]): Args = + value.map(append(_)).getOrElse(this) + + def appendOptionVal(first: String, second: Option[String]): Args = + second.map(b => append(first, b)).getOrElse(this) + + def appendWhen(flag: Boolean)(a: String, as: String*): Args = + if (flag) append(a, as: _*) else this + + def appendWhenNot(flag: Boolean)(a: String, as: String*): Args = + if (!flag) append(a, as: _*) else this + + def append(p: Path): Args = + append(p.toString) + + def append(as: Iterable[String]): Args = + Args(args ++ as.toVector) + } + object Args { + val empty: Args = Args() + + def apply(as: String*): Args = + Args(as.toVector) + } + final case class Result(rc: Int, stdout: String, stderr: String) def exec[F[_]: Sync]( @@ -104,6 +167,10 @@ object SystemCommand { .redirectError(Redirect.PIPE) .redirectOutput(Redirect.PIPE) + val pbEnv = pb.environment() + cmd.env.foreach { case (key, value) => + pbEnv.put(key, value) + } wd.map(_.toNioPath.toFile).foreach(pb.directory) pb.start() } diff --git a/modules/common/src/main/scala/docspell/common/UrlMatcher.scala b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala new file mode 100644 index 00000000..c8fd393e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList +import cats.kernel.Monoid +import cats.syntax.all._ + +trait UrlMatcher { + def matches(url: LenientUri): Boolean +} + +object UrlMatcher { + val True = instance(_ => true) + val False = instance(_ => false) + + def instance(f: LenientUri => Boolean): UrlMatcher = + (url: LenientUri) => f(url) + + def fromString(str: String): Either[String, UrlMatcher] = + if (str == "") False.asRight + else if (str == "*") True.asRight + else LenientUri.parse(str).map(fromUrl) + + def unsafeFromString(str: String): UrlMatcher = + fromString(str).fold(sys.error, identity) + + def fromStringList(str: List[String]): Either[String, UrlMatcher] = + str match { + case Nil => False.asRight + case _ => str.map(_.trim).traverse(fromString).map(_.combineAll) + } + + def fromUrl(url: LenientUri): UrlMatcher = { + val schemeGlob = Glob(url.scheme.head) + val hostGlob = HostGlob(url.host) + val pathGlob = Glob(url.path.asString) + new Impl(schemeGlob, hostGlob, pathGlob, url.path.segments.size) + } + + def any(ulrm: IterableOnce[UrlMatcher]): UrlMatcher = + anyMonoid.combineAll(ulrm) + + def all(urlm: IterableOnce[UrlMatcher]): UrlMatcher = + allMonoid.combineAll(urlm) + + val anyMonoid: Monoid[UrlMatcher] = + Monoid.instance(False, (a, b) => instance(url => a.matches(url) || b.matches(url))) + + val allMonoid: Monoid[UrlMatcher] = + Monoid.instance(True, (a, b) => instance(url => a.matches(url) && b.matches(url))) + + implicit val defaultMonoid: Monoid[UrlMatcher] = anyMonoid + + private class Impl(scheme: Glob, host: HostGlob, path: Glob, pathSegmentCount: Int) + extends UrlMatcher { + def matches(url: LenientUri) = { + // strip path to only match prefixes + val mPath: LenientUri.Path = + NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match { + case Some(nel) => LenientUri.NonEmptyPath(nel) + case None => LenientUri.RootPath + } + + url.scheme.forall(scheme.matches(false)) && + host.matches(url.host) && + path.matchFilenameOrPath(mPath.asString) + } + } + + private class HostGlob(glob: Option[Glob]) { + def matches(host: Option[String]): Boolean = + (glob, host) match { + case (Some(pattern), Some(word)) => + pattern.matches(false)(HostGlob.prepareHost(word)) + case (None, None) => true + case _ => false + } + + override def toString = s"HostGlob(${glob.map(_.asString)})" + } + + private object HostGlob { + def apply(hostPattern: Option[String]): HostGlob = + new HostGlob(hostPattern.map(p => Glob(prepareHost(p)))) + + private def prepareHost(host: String): String = + host.replace('.', '/') + } +} diff --git a/modules/common/src/main/scala/docspell/common/UrlReader.scala b/modules/common/src/main/scala/docspell/common/UrlReader.scala new file mode 100644 index 00000000..7e1521f2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlReader.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.ApplicativeError +import cats.effect._ +import fs2.Stream + +trait UrlReader[F[_]] { + def apply(url: LenientUri): Stream[F, Byte] +} + +object UrlReader { + + def instance[F[_]](f: LenientUri => Stream[F, Byte]): UrlReader[F] = + (url: LenientUri) => f(url) + + def failWith[F[_]]( + message: String + )(implicit F: ApplicativeError[F, Throwable]): UrlReader[F] = + instance(url => + Stream.raiseError( + new IllegalStateException(s"Unable to read '${url.asString}': $message") + ) + ) + + def apply[F[_]](implicit r: UrlReader[F]): UrlReader[F] = r + + implicit def defaultReader[F[_]: Sync]: UrlReader[F] = + instance(_.readURL[F](8192)) +} diff --git a/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala new file mode 100644 index 00000000..f0caa3a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AttachmentAction {} + +object AttachmentAction { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class SetExtractedText(text: Option[String]) extends AttachmentAction + object SetExtractedText { + implicit val jsonDecoder: Decoder[SetExtractedText] = deriveDecoder + implicit val jsonEncoder: Encoder[SetExtractedText] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[AttachmentAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AttachmentAction] = deriveConfiguredEncoder + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala new file mode 100644 index 00000000..f5732f64 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait BackendCommand {} + +object BackendCommand { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("command").withKebabCaseConstructorNames + + case class ItemUpdate(itemId: Ident, actions: List[ItemAction]) extends BackendCommand + object ItemUpdate { + implicit val jsonDecoder: Decoder[ItemUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemUpdate] = deriveEncoder + } + + def item(itemId: Ident, actions: List[ItemAction]): BackendCommand = + ItemUpdate(itemId, actions) + + case class AttachmentUpdate( + itemId: Ident, + attachId: Ident, + actions: List[AttachmentAction] + ) extends BackendCommand + object AttachmentUpdate { + implicit val jsonDecoder: Decoder[AttachmentUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachmentUpdate] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[BackendCommand] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[BackendCommand] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala new file mode 100644 index 00000000..863e35ed --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +trait BackendCommandRunner[F[_], A] { + + def run(collective: Ident, cmd: BackendCommand): F[A] + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[A] + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala new file mode 100644 index 00000000..16a09048 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait ItemAction {} + +object ItemAction { + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class AddTags(tags: Set[String]) extends ItemAction + object AddTags { + implicit val jsonDecoder: Decoder[AddTags] = deriveDecoder + implicit val jsonEncoder: Encoder[AddTags] = deriveEncoder + } + + case class ReplaceTags(tags: Set[String]) extends ItemAction + object ReplaceTags { + implicit val jsonDecoder: Decoder[ReplaceTags] = deriveDecoder + implicit val jsonEncoder: Encoder[ReplaceTags] = deriveEncoder + } + + case class RemoveTags(tags: Set[String]) extends ItemAction + object RemoveTags { + implicit val jsonDecoder: Decoder[RemoveTags] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTags] = deriveEncoder + } + + case class RemoveTagsCategory(categories: Set[String]) extends ItemAction + object RemoveTagsCategory { + implicit val jsonDecoder: Decoder[RemoveTagsCategory] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTagsCategory] = deriveEncoder + } + + case class SetFolder(folder: Option[String]) extends ItemAction + object SetFolder { + implicit val jsonDecoder: Decoder[SetFolder] = deriveDecoder + implicit val jsonEncoder: Encoder[SetFolder] = deriveEncoder + } + + case class SetCorrOrg(id: Option[Ident]) extends ItemAction + object SetCorrOrg { + implicit val jsonDecoder: Decoder[SetCorrOrg] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrOrg] = deriveEncoder + } + + case class SetCorrPerson(id: Option[Ident]) extends ItemAction + object SetCorrPerson { + implicit val jsonDecoder: Decoder[SetCorrPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrPerson] = deriveEncoder + } + + case class SetConcPerson(id: Option[Ident]) extends ItemAction + object SetConcPerson { + implicit val jsonDecoder: Decoder[SetConcPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcPerson] = deriveEncoder + } + + case class SetConcEquipment(id: Option[Ident]) extends ItemAction + object SetConcEquipment { + implicit val jsonDecoder: Decoder[SetConcEquipment] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcEquipment] = deriveEncoder + } + + case class SetField(field: Ident, value: String) extends ItemAction + object SetField { + implicit val jsonDecoder: Decoder[SetField] = deriveDecoder + implicit val jsonEncoder: Encoder[SetField] = deriveEncoder + } + + case class SetName(name: String) extends ItemAction + object SetName { + implicit val jsonDecoder: Decoder[SetName] = deriveDecoder + implicit val jsonEncoder: Encoder[SetName] = deriveEncoder + } + + case class SetNotes(notes: Option[String]) extends ItemAction + object SetNotes { + implicit val jsonDecoder: Decoder[SetNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[SetNotes] = deriveEncoder + } + + case class AddNotes(notes: String, separator: Option[String]) extends ItemAction + object AddNotes { + implicit val jsonDecoder: Decoder[AddNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[AddNotes] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[ItemAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[ItemAction] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Args.scala b/modules/common/src/main/scala/docspell/common/exec/Args.scala new file mode 100644 index 00000000..292899a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Args.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import fs2.io.file.Path + +case class Args(values: Seq[String]) { + + def option(key: String, value: String): Args = + Args(values ++ Seq(key, value)) + + def option(key: String, value: Option[String]): Args = + value.map(v => option(key, v)).getOrElse(this) + + def appendOpt(v: Option[String]): Args = + v.map(e => Args(values :+ e)).getOrElse(this) + + def append(v: String, vs: String*): Args = + Args(values ++ (v +: vs)) + + def append(path: Path): Args = + append(path.toString) + + def append(args: Args): Args = + Args(values ++ args.values) + + def append(args: Seq[String]): Args = + Args(values ++ args) + + def prepend(v: String): Args = + Args(v +: values) + + def prependWhen(flag: Boolean)(v: String) = + if (flag) prepend(v) else this + + def cmdString: String = + values.mkString(" ") +} + +object Args { + val empty: Args = Args(Seq.empty) + + def of(v: String*): Args = + Args(v) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Env.scala b/modules/common/src/main/scala/docspell/common/exec/Env.scala new file mode 100644 index 00000000..2524d35a --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Env.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +case class Env(values: Map[String, String]) { + + def add(name: String, value: String): Env = + copy(values.updated(name, value)) + + def addAll(v: Map[String, String]): Env = + Env(values ++ v) + + def addAll(e: Env): Env = + Env(values ++ e.values) + + def ++(e: Env) = addAll(e) + + def foreach(f: (String, String) => Unit): Unit = + values.foreach(t => f(t._1, t._2)) + + def map[A](f: (String, String) => A): Seq[A] = + values.map(f.tupled).toSeq + + def mapConcat[A](f: (String, String) => Seq[A]): Seq[A] = + values.flatMap(f.tupled).toSeq +} + +object Env { + val empty: Env = Env(Map.empty) + + def of(nv: (String, String)*): Env = + Env(Map(nv: _*)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala new file mode 100644 index 00000000..b94b8c4b --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import docspell.common._ + +final case class SysCmd( + program: String, + args: Args, + env: Env, + timeout: Duration +) { + + def withArgs(f: Args => Args): SysCmd = + copy(args = f(args)) + + def withTimeout(to: Duration): SysCmd = + copy(timeout = to) + + def withEnv(f: Env => Env): SysCmd = + copy(env = f(env)) + + def addEnv(env: Env): SysCmd = + withEnv(_.addAll(env)) + + def cmdString: String = + s"$program ${args.cmdString}" + + private[exec] def toCmd: Seq[String] = + program +: args.values +} + +object SysCmd { + def apply(prg: String, args: String*): SysCmd = + apply(prg, Args(args)) + + def apply(prg: String, args: Args): SysCmd = + SysCmd(prg, args, Env.empty, Duration.minutes(2)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysExec.scala b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala new file mode 100644 index 00000000..da7b10c2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala @@ -0,0 +1,163 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import java.lang.ProcessBuilder.Redirect +import java.util.concurrent.TimeUnit + +import scala.concurrent.TimeoutException +import scala.jdk.CollectionConverters._ + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Path +import fs2.{Pipe, Stream} + +import docspell.common.Duration +import docspell.logging.Logger + +trait SysExec[F[_]] { + + def stdout: Stream[F, Byte] + + def stdoutLines: Stream[F, String] = + stdout + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def stderr: Stream[F, Byte] + + def stderrLines: Stream[F, String] = + stderr + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def waitFor(timeout: Option[Duration] = None): F[Int] + + /** Sends a signal to the process to terminate it immediately */ + def cancel: F[Unit] + + /** Consume lines of output of the process in background. */ + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] + + /** Consumes stderr lines (left) and stdout lines (right) in a background thread. */ + def consumeOutputs( + m: Either[String, String] => F[Unit] + )(implicit F: Async[F]): Resource[F, SysExec[F]] = { + val pe: Pipe[F, String, Unit] = _.map(_.asLeft).evalMap(m) + val po: Pipe[F, String, Unit] = _.map(_.asRight).evalMap(m) + consumeOutputs(po, pe) + } + + def logOutputs(logger: Logger[F], name: String)(implicit F: Async[F]) = + consumeOutputs { + case Right(line) => logger.debug(s"[$name (out)]: $line") + case Left(line) => logger.debug(s"[$name (err)]: $line") + } +} + +object SysExec { + private val readChunkSz = 8 * 1024 + + def apply[F[_]: Sync]( + cmd: SysCmd, + logger: Logger[F], + workdir: Option[Path] = None, + stdin: Option[Stream[F, Byte]] = None + ): Resource[F, SysExec[F]] = + for { + proc <- startProcess(logger, cmd, workdir, stdin) + fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil)) + } yield new SysExec[F] { + def stdout: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getInputStream), + readChunkSz, + closeAfterUse = false + ) + + def stderr: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getErrorStream), + readChunkSz, + closeAfterUse = false + ) + + def cancel = Sync[F].blocking(proc.destroy()) + + def waitFor(timeout: Option[Duration]): F[Int] = { + val to = timeout.getOrElse(cmd.timeout) + logger.trace("Waiting for command to terminate…") *> + Sync[F] + .blocking(proc.waitFor(to.millis, TimeUnit.MILLISECONDS)) + .flatTap(_ => fibers.get.flatMap(_.traverse_(identity))) + .flatMap(terminated => + if (terminated) proc.exitValue().pure[F] + else + Sync[F] + .raiseError( + new TimeoutException(s"Timed out after: ${to.formatExact}") + ) + ) + } + + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] = + for { + f1 <- F.background(stdoutLines.through(out).compile.drain) + f2 <- F.background(stderrLines.through(err).compile.drain) + _ <- Resource.eval(fibers.update(list => f1.void :: f2.void :: list)) + } yield this + } + + private def startProcess[F[_]: Sync, A]( + logger: Logger[F], + cmd: SysCmd, + workdir: Option[Path], + stdin: Option[Stream[F, Byte]] + ): Resource[F, Process] = { + val log = logger.debug(s"Running external command: ${cmd.cmdString}") + + val proc = log *> + Sync[F].blocking { + val pb = new ProcessBuilder(cmd.toCmd.asJava) + .redirectInput(if (stdin.isDefined) Redirect.PIPE else Redirect.INHERIT) + .redirectError(Redirect.PIPE) + .redirectOutput(Redirect.PIPE) + + val pbEnv = pb.environment() + cmd.env.foreach { (name, v) => + pbEnv.put(name, v) + () + } + workdir.map(_.toNioPath.toFile).foreach(pb.directory) + pb.start() + } + + Resource + .make(proc)(p => + logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy()) + ) + .evalMap(p => + stdin match { + case Some(in) => + writeToProcess(in, p).compile.drain.as(p) + case None => + p.pure[F] + } + ) + } + + private def writeToProcess[F[_]: Sync]( + data: Stream[F, Byte], + proc: Process + ): Stream[F, Nothing] = + data.through(fs2.io.writeOutputStream(Sync[F].blocking(proc.getOutputStream))) +} diff --git a/modules/common/src/main/scala/docspell/common/File.scala b/modules/common/src/main/scala/docspell/common/util/File.scala similarity index 91% rename from modules/common/src/main/scala/docspell/common/File.scala rename to modules/common/src/main/scala/docspell/common/util/File.scala index 0d6505dd..679c99ec 100644 --- a/modules/common/src/main/scala/docspell/common/File.scala +++ b/modules/common/src/main/scala/docspell/common/util/File.scala @@ -4,20 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.common +package docspell.common.util import java.nio.file.{Path => JPath} -import cats.FlatMap -import cats.Monad import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import cats.{FlatMap, Monad} import fs2.Stream import fs2.io.file.{Files, Flags, Path} -import docspell.common.syntax.all._ - import io.circe.Decoder +import io.circe.parser object File { @@ -75,6 +73,5 @@ object File { .map(_ => file) def readJson[F[_]: Async, A](file: Path)(implicit d: Decoder[A]): F[A] = - readText[F](file).map(_.parseJsonAs[A]).rethrow - + readText[F](file).map(parser.decode[A]).rethrow } diff --git a/modules/common/src/main/scala/docspell/common/util/Random.scala b/modules/common/src/main/scala/docspell/common/util/Random.scala new file mode 100644 index 00000000..b192aa6d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/util/Random.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.util + +import cats.effect._ + +import scodec.bits.ByteVector + +trait Random[F[_]] { + def string(len: Int): F[String] + def string: F[String] = string(8) +} + +object Random { + def apply[F[_]: Sync] = + new Random[F] { + def string(len: Int) = Sync[F].delay { + val buf = Array.ofDim[Byte](len) + new scala.util.Random().nextBytes(buf) + ByteVector.view(buf).toBase58 + } + } +} diff --git a/modules/common/src/test/scala/docspell/common/GlobTest.scala b/modules/common/src/test/scala/docspell/common/GlobTest.scala index 7e3d3bf8..4ee7d468 100644 --- a/modules/common/src/test/scala/docspell/common/GlobTest.scala +++ b/modules/common/src/test/scala/docspell/common/GlobTest.scala @@ -70,11 +70,13 @@ class GlobTest extends FunSuite { test("with splitting") { assert(Glob("a/b/*").matches(true)("a/b/hello")) + assert(!Glob("a/b/*").matches(true)("a/b/hello/bello")) assert(!Glob("a/b/*").matches(true)("/a/b/hello")) assert(Glob("/a/b/*").matches(true)("/a/b/hello")) assert(!Glob("/a/b/*").matches(true)("a/b/hello")) assert(!Glob("*/a/b/*").matches(true)("a/b/hello")) assert(Glob("*/a/b/*").matches(true)("test/a/b/hello")) + assert(!Glob("/a/b").matches(true)("/a/b/c/d")) } test("asString") { diff --git a/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala new file mode 100644 index 00000000..51fb1f19 --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import munit._ + +class UrlMatcherTest extends FunSuite { + + test("it should match patterns") { + assertUrlsMatch( + uri("https://github.com/docspell/*") -> uri("https://github.com/docspell/dsc"), + uri("*s://test.com/*") -> uri("https://test.com/a"), + uri("*s://test.com/*") -> uri("https://test.com/a/b"), + uri("*s://test.com/*") -> uri("https://test.com/a/b/c"), + uri("*s://test.com/project/*") -> uri("https://test.com/project/c"), + uri("https://*.test.com/projects/*") -> uri("https://a.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1") + ) + + assertUrlsNotMatch( + uri("https://*.test.com/projects/*") -> uri("https://test.com/projects/p1"), + uri("*s://test.com/project/*") -> uri("https://test.com/subject/c") + ) + } + + def uri(str: String): LenientUri = LenientUri.unsafe(str) + + def assertUrlsMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri does not match $checkUri" + ) + } + + def assertUrlsMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsMatch(test :: more.toList) + + def assertUrlsNotMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + !UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri incorrectly matches $checkUri" + ) + } + + def assertUrlsNotMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsNotMatch(test :: more.toList) +} diff --git a/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala new file mode 100644 index 00000000..bb0cfaef --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common._ + +import io.circe.parser +import io.circe.syntax._ +import munit._ + +class BackendCommandTest extends FunSuite { + + test("encode json") { + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals( + bc.asJson.spaces2, + """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + ) + } + + test("decode case insensitive keys") { + val json = """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals(parser.decode[BackendCommand](json), Right(bc)) + } + + def id(str: String) = Ident.unsafe(str) +} diff --git a/modules/config/src/main/scala/docspell/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala index e529a2c5..014643ae 100644 --- a/modules/config/src/main/scala/docspell/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -13,6 +13,7 @@ import scala.reflect.ClassTag import cats.syntax.all._ import fs2.io.file.Path +import docspell.addons.RunnerType import docspell.common._ import docspell.ftspsql.{PgQueryParser, RankNormalization} import docspell.logging.{Level, LogConfig} @@ -32,6 +33,17 @@ object Implicits { else super.fieldValue(name) } + implicit val urlMatcherReader: ConfigReader[UrlMatcher] = { + val fromList = ConfigReader[List[String]].emap(reason(UrlMatcher.fromStringList)) + val fromString = ConfigReader[String].emap( + reason(str => UrlMatcher.fromStringList(str.split("[\\s,]+").toList)) + ) + fromList.orElse(fromString) + } + + implicit val runnerSelectReader: ConfigReader[List[RunnerType]] = + ConfigReader[String].emap(reason(RunnerType.fromSeparatedString)) + implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) diff --git a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala index eff5a0fb..8f0f9e11 100644 --- a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala +++ b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala @@ -12,6 +12,7 @@ import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt} import docspell.logging.Logger diff --git a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala index 25905afe..06ee81c1 100644 --- a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala @@ -15,6 +15,7 @@ import cats.implicits._ import fs2.Stream import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.convert.extern.OcrMyPdfConfig import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig} diff --git a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala index 96f251ff..ad41c01c 100644 --- a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala +++ b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala @@ -18,6 +18,7 @@ import fs2.io.file.Path import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.files.TikaMimetype diff --git a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala index 6f0ab2ab..9beaed28 100644 --- a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala @@ -14,6 +14,7 @@ import cats.effect.unsafe.implicits.global import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.convert._ import docspell.files.ExampleFiles import docspell.logging.TestLoggingConfig diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala index 727666a8..f39e4b0b 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala @@ -11,6 +11,7 @@ import fs2.Stream import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger object Ocr { diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala index 003fa8c3..856c21a3 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala @@ -11,6 +11,7 @@ import java.nio.file.Paths import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File case class OcrConfig( maxImageSize: Int, diff --git a/modules/files/src/main/scala/docspell/files/FileSupport.scala b/modules/files/src/main/scala/docspell/files/FileSupport.scala new file mode 100644 index 00000000..5219ac37 --- /dev/null +++ b/modules/files/src/main/scala/docspell/files/FileSupport.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.files + +import cats.data.OptionT +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.{MimeType, MimeTypeHint} + +import io.circe.Encoder +import io.circe.syntax._ + +trait FileSupport { + implicit final class FileOps[F[_]: Files: Sync](self: Path) { + def detectMime: F[Option[MimeType]] = + Files[F].isReadable(self).flatMap { flag => + OptionT + .whenF(flag) { + TikaMimetype + .detect( + Files[F].readAll(self), + MimeTypeHint.filename(self.fileName.toString) + ) + } + .value + } + + def asTextFile(alt: MimeType => F[Unit]): F[Option[Path]] = + OptionT(detectMime).flatMapF { mime => + if (mime.matches(MimeType.text("plain"))) self.some.pure[F] + else alt(mime).as(None: Option[Path]) + }.value + + def readText: F[String] = + Files[F] + .readAll(self) + .through(fs2.text.utf8.decode) + .compile + .string + + def readAll: Stream[F, Byte] = + Files[F].readAll(self) + + def writeJson[A: Encoder](value: A): F[Unit] = + Stream + .emit(value.asJson.noSpaces) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(self)) + .compile + .drain + } +} + +object FileSupport extends FileSupport diff --git a/modules/files/src/main/scala/docspell/files/Zip.scala b/modules/files/src/main/scala/docspell/files/Zip.scala index 3fd938e4..b8d1dfd0 100644 --- a/modules/files/src/main/scala/docspell/files/Zip.scala +++ b/modules/files/src/main/scala/docspell/files/Zip.scala @@ -8,11 +8,12 @@ package docspell.files import java.io.InputStream import java.nio.charset.StandardCharsets -import java.nio.file.Paths import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} +import cats.data.OptionT import cats.effect._ import cats.implicits._ +import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common.Binary @@ -27,16 +28,72 @@ object Zip { ): Pipe[F, (String, Stream[F, Byte]), Byte] = in => zipJava(logger, chunkSize, in.through(deduplicate)) - def unzipP[F[_]: Async](chunkSize: Int, glob: Glob): Pipe[F, Byte, Binary[F]] = - s => unzip[F](chunkSize, glob)(s) + def unzip[F[_]: Async]( + chunkSize: Int, + glob: Glob + ): Pipe[F, Byte, Binary[F]] = + s => unzipStream[F](chunkSize, glob)(s) - def unzip[F[_]: Async](chunkSize: Int, glob: Glob)( + def unzipStream[F[_]: Async](chunkSize: Int, glob: Glob)( data: Stream[F, Byte] ): Stream[F, Binary[F]] = data .through(fs2.io.toInputStream[F]) .flatMap(in => unzipJava(in, chunkSize, glob)) + def saveTo[F[_]: Async]( + logger: Logger[F], + targetDir: Path, + moveUp: Boolean + ): Pipe[F, Binary[F], Path] = + binaries => + binaries + .filter(e => !e.name.endsWith("/")) + .evalMap { entry => + val out = targetDir / entry.name + val createParent = + OptionT + .fromOption[F](out.parent) + .flatMapF(parent => + Files[F] + .exists(parent) + .map(flag => Option.when(!flag)(parent)) + ) + .semiflatMap(p => Files[F].createDirectories(p)) + .getOrElse(()) + + logger.trace(s"Unzip ${entry.name} -> $out") *> + createParent *> + entry.data.through(Files[F].writeAll(out)).compile.drain + } + .drain ++ Stream + .eval(if (moveUp) moveContentsUp(logger)(targetDir) else ().pure[F]) + .as(targetDir) + + private def moveContentsUp[F[_]: Sync: Files](logger: Logger[F])(dir: Path): F[Unit] = + Files[F] + .list(dir) + .take(2) + .compile + .toList + .flatMap { + case subdir :: Nil => + Files[F].isDirectory(subdir).flatMap { + case false => ().pure[F] + case true => + Files[F] + .list(subdir) + .filter(p => p != dir) + .evalTap(c => logger.trace(s"Move $c -> ${dir / c.fileName}")) + .evalMap(child => Files[F].move(child, dir / child.fileName)) + .compile + .drain + } + + case _ => + ().pure[F] + } + def unzipJava[F[_]: Async]( in: InputStream, chunkSize: Int, @@ -55,7 +112,7 @@ object Zip { .unNoneTerminate .filter(ze => glob.matchFilenameOrPath(ze.getName())) .map { ze => - val name = Paths.get(ze.getName()).getFileName.toString + val name = ze.getName() val data = fs2.io.readInputStream[F]((zin: InputStream).pure[F], chunkSize, false) Binary(name, data) diff --git a/modules/files/src/test/resources/zip-dirs-one.zip b/modules/files/src/test/resources/zip-dirs-one.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a68b33aed12473ef2ba6f817e5c32ecd82b3d4e GIT binary patch literal 1306 zcmajfK}y3w6b9f)lZZx;fExu77orQHwNYHS5picX5i0HkwG|5yMIux;B0Yo`5O3iT z1drjyJNW*YO!|_UF)5Q~k@@o8%TLmtA4CGj`o&vcUKFQLtcd6Q_P%r4pA-Jk_thoV z<9J?p!H+LQFrXXLa{Fd&d!)AyMh{(O9MgU*r?FKt;)(&NkWNU!vPvzg3)0&5{@x5- zV=Y^x>rpn`Y3JiyHs~O4hFNwOsWt(lO%5+sV30{kI{M`#iK1QWnollKlRKkcU;gVC zhO4ZkgR*wZvVdXLhR4f7r^o{*7j&o4x~1j_hSt2+s_)9z61t}&7Qcu1FVYsOwAIQ> zdsE8H8mA64RV0r*YwD(+nLF-HDxOXusn!H4)t*76dgQ&oepytO$Y$P5BIh^gDRQ22 z2rQS)(=hYMr}4yKq?0^LnDpDi9L#L;LO&}WOKLN0CX~Z5JzGwI<&{k9M=ZoQLd??d KgLa*xYVijgEA^!S literal 0 HcmV?d00001 diff --git a/modules/files/src/test/resources/zip-dirs.zip b/modules/files/src/test/resources/zip-dirs.zip new file mode 100644 index 0000000000000000000000000000000000000000..816f8413ad768f4c3220a3f37d425386b03bb768 GIT binary patch literal 1098 zcmWIWW@h1H00H&H=^C#YQh*8r z;M%p(v~$C>gUo;$5(qQ^gs~dJ1T+q0h=PHY85@ud!Xl`Kq-Ex$8tau*lz^QCGN~TZ zw2ai8oO~{b!$4+&{Do{LAIwaUX-06fjb6=Ip|(fK{JgL=rxc*hH!(Nu{#K65F?WuGp^W|0C^V(1Q^~rf@o-3V1=Xw zj3`7Hg_#nNjk1Rs1xpt|1F@zFh=JHs1;ju`pixU2TTl%|N+X0##!MT?Cf|UWjFeIc zdml4}AbVd8=xCtNLFol(6eyl?ge1u0n5hNXd8sg?kkSreGcnT)vYDHKW+J5^pmCrS WgyBS1Q08P{2SN*=JO6+pf&l=Ev&Ap~ literal 0 HcmV?d00001 diff --git a/modules/files/src/test/scala/docspell/files/ZipTest.scala b/modules/files/src/test/scala/docspell/files/ZipTest.scala index 6012e1c2..9230ac8f 100644 --- a/modules/files/src/test/scala/docspell/files/ZipTest.scala +++ b/modules/files/src/test/scala/docspell/files/ZipTest.scala @@ -7,20 +7,25 @@ package docspell.files import cats.effect._ -import cats.effect.unsafe.implicits.global import cats.implicits._ +import fs2.io.file.{Files, Path} import docspell.common.Glob +import docspell.logging.TestLoggingConfig import munit._ -class ZipTest extends FunSuite { +class ZipTest extends CatsEffectSuite with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + val tempDir = ResourceFixture( + Files[IO].tempDirectory(Path("target").some, "zip-test-", None) + ) test("unzip") { val zipFile = ExampleFiles.letters_zip.readURL[IO](8192) - val uncomp = zipFile.through(Zip.unzip(8192, Glob.all)) + val unzip = zipFile.through(Zip.unzip(8192, Glob.all)) - uncomp + unzip .evalMap { entry => val x = entry.data.map(_ => 1).foldMonoid.compile.lastOrError x.map { size => @@ -35,6 +40,10 @@ class ZipTest extends FunSuite { } .compile .drain - .unsafeRunSync() + } + + tempDir.test("unzipTo directory tree") { _ => + // val zipFile = ExampleFiles.zip_dirs_zip.readURL[IO](8192) + // zipFile.through(Zip.unzip(G)) } } diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 780d48b4..7a7d9b97 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -780,4 +780,75 @@ Docpell Update Check index-all-chunk = 10 } } + + addons { + # A directory to extract addons when running them. Everything in + # here will be cleared after each run. + working-dir = ${java.io.tmpdir}"/docspell-addons" + + # A directory for addons to store data between runs. This is not + # cleared by Docspell and can get large depending on the addons + # executed. + # + # This directory is used as base. In it subdirectories are created + # per run configuration id. + cache-dir = ${java.io.tmpdir}"/docspell-addon-cache" + + executor-config { + # Define a (comma or whitespace separated) list of runners that + # are responsible for executing an addon. This setting is + # compared to what is supported by addons. Possible values are: + # + # - nix-flake: use nix-flake runner if the addon supports it + # (this requires the nix package manager on the joex machine) + # - docker: use docker + # - trivial: use the trivial runner + # + # The first successful execution is used. This should list all + # runners the computer supports. + runner = "nix-flake, docker, trivial" + + # systemd-nspawn can be used to run the program in a container. + # This is used by runners nix-flake and trivial. + nspawn = { + # If this is false, systemd-nspawn is not tried. When true, the + # addon is executed inside a lightweight container via + # systemd-nspawn. + enabled = false + + # Path to sudo command. By default systemd-nspawn is executed + # via sudo - the user running joex must be allowed to do so NON + # INTERACTIVELY. If this is empty, then nspawn is tried to + # execute without sudo. + sudo-binary = "sudo" + + # Path to the systemd-nspawn command. + nspawn-binary = "systemd-nspawn" + + # Workaround, if multiple same named containers are run too fast + container-wait = "100 millis" + } + + # The timeout for running an addon. + run-timeout = "15 minutes" + + # Configure the nix flake runner. + nix-runner { + # Path to the nix command. + nix-binary = "nix" + + # The timeout for building the package (running nix build). + build-timeout = "15 minutes" + } + + # Configure the docker runner + docker-runner { + # Path to the docker command. + docker-binary = "docker" + + # The timeout for building the package (running docker build). + build-timeout = "15 minutes" + } + } + } } \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index de171135..646033c2 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.TextAnalysisConfig import docspell.analysis.classifier.TextClassifierConfig import docspell.backend.Config.Files +import docspell.backend.joex.AddonEnvConfig import docspell.common._ import docspell.config.{FtsType, PgFtsConfig} import docspell.convert.ConvertConfig @@ -43,7 +44,8 @@ case class Config( files: Files, mailDebug: Boolean, fullTextSearch: Config.FullTextSearch, - updateCheck: UpdateCheckConfig + updateCheck: UpdateCheckConfig, + addons: AddonEnvConfig ) { def pubSubConfig(headerValue: Ident): PubSubConfig = diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index c410cc42..e3a679a8 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -145,6 +145,8 @@ object JoexAppImpl extends MailAddressCodec { schedulerModule.scheduler, schedulerModule.periodicScheduler ) + nodes <- ONode(store) + _ <- nodes.withRegistered(cfg.appId, NodeType.Joex, cfg.baseUrl, None) appR <- Resource.make(app.init.map(_ => app))(_.initShutdown) } yield appR diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 5bdc8f18..ae1bf1a7 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -59,7 +59,7 @@ object JoexServer { Router("pubsub" -> pubSub.receiveRoute) }, "/api/info" -> InfoRoutes(cfg), - "/api/v1" -> JoexRoutes(joexApp) + "/api/v1" -> JoexRoutes(cfg, joexApp) ).orNotFound // With Middlewares in place diff --git a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala index 913ee930..f5c99d47 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala @@ -9,7 +9,9 @@ package docspell.joex import cats.effect.{Async, Resource} import docspell.analysis.TextAnalyser +import docspell.backend.BackendCommands import docspell.backend.fulltext.CreateIndex +import docspell.backend.joex.AddonOps import docspell.backend.ops._ import docspell.backend.task.DownloadZipArgs import docspell.common._ @@ -17,6 +19,7 @@ import docspell.config.FtsType import docspell.ftsclient.FtsClient import docspell.ftspsql.PsqlFtsClient import docspell.ftssolr.SolrFtsClient +import docspell.joex.addon.{ItemAddonTask, ScheduledAddonTask} import docspell.joex.analysis.RegexNerFile import docspell.joex.download.DownloadZipTask import docspell.joex.emptytrash.EmptyTrashTask @@ -32,6 +35,7 @@ import docspell.joex.preview.{AllPreviewsTask, MakePreviewTask} import docspell.joex.process.{ItemHandler, ReProcessItem} import docspell.joex.scanmailbox.ScanMailboxTask import docspell.joex.updatecheck.{ThisVersion, UpdateCheck, UpdateCheckTask} +import docspell.joexapi.client.JoexClient import docspell.notification.api.NotificationModule import docspell.pubsub.api.PubSubT import docspell.scheduler.impl.JobStoreModuleBuilder @@ -57,7 +61,8 @@ final class JoexTasks[F[_]: Async]( createIndex: CreateIndex[F], joex: OJoex[F], jobs: OJob[F], - itemSearch: OItemSearch[F] + itemSearch: OItemSearch[F], + addons: AddonOps[F] ) { val downloadAll: ODownloadAll[F] = ODownloadAll(store, jobs, jobStoreModule.jobs) @@ -68,7 +73,8 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, store, itemOps, fts, analyser, regexNer), + ItemHandler + .newItem[F](cfg, store, itemOps, fts, analyser, regexNer, addons), ItemHandler.onCancel[F](store) ) ) @@ -82,7 +88,15 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), + ReProcessItem[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addons, + store + ), ReProcessItem.onCancel[F] ) ) @@ -223,6 +237,20 @@ final class JoexTasks[F[_]: Async]( DownloadZipTask.onCancel[F] ) ) + .withTask( + JobTask.json( + ScheduledAddonTaskArgs.taskName, + ScheduledAddonTask[F](addons), + ScheduledAddonTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ItemAddonTaskArgs.taskName, + ItemAddonTask[F](addons, store), + ItemAddonTask.onCancel[F] + ) + ) } object JoexTasks { @@ -237,8 +265,9 @@ object JoexTasks { emailService: Emil[F] ): Resource[F, JoexTasks[F]] = for { - joex <- OJoex(pubSub) - store = jobStoreModule.store + store <- Resource.pure(jobStoreModule.store) + node <- ONode(store) + joex <- OJoex(pubSub, node, JoexClient(httpClient)) upload <- OUpload(store, jobStoreModule.jobs) fts <- createFtsClient(cfg, pools, store, httpClient) createIndex <- CreateIndex.resource(fts, store) @@ -250,6 +279,16 @@ object JoexTasks { notification <- ONotification(store, notificationModule) fileRepo <- OFileRepository(store, jobStoreModule.jobs) jobs <- OJob(store, joex, pubSub) + fields <- OCustomFields(store) + attachmentOps = OAttachment(store, fts, jobStoreModule.jobs) + cmdRunner = BackendCommands(itemOps, attachmentOps, fields, notification, None) + addons = AddonOps( + cfg.addons, + store, + cmdRunner, + attachmentOps, + jobStoreModule.jobs + ) } yield new JoexTasks[F]( cfg, store, @@ -266,7 +305,8 @@ object JoexTasks { createIndex, joex, jobs, - itemSearchOps + itemSearchOps, + addons ) private def createFtsClient[F[_]: Async]( diff --git a/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala new file mode 100644 index 00000000..0b1bad57 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.MonadError + +import docspell.addons.AddonExecutionResult +import docspell.scheduler.PermanentError + +trait AddonTaskExtension { + implicit final class AddonExecutionResultOps(self: AddonExecutionResult) { + def raiseErrorIfNeeded[F[_]](implicit m: MonadError[F, Throwable]): F[Unit] = + if (self.isFailure && self.pure) { + m.raiseError(new Exception(s"Addon execution failed: $self")) + } else if (self.isFailure) { + m.raiseError( + PermanentError( + new Exception( + "Addon execution failed. Do not retry, because some addons were impure." + ) + ) + ) + } else m.pure(()) + + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala new file mode 100644 index 00000000..50bbf41b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Files + +import docspell.addons.{AddonTriggerType, InputEnv, Middleware} +import docspell.backend.joex.AddonOps.ExecResult +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.common._ +import docspell.files.FileSupport +import docspell.joex.process.ItemData +import docspell.logging.Logger +import docspell.scheduler.Task +import docspell.store.Store +import docspell.store.queries.QAttachment + +object GenericItemAddonTask extends LoggerExtension with FileSupport { + + private val itemSubdir = "item" + private val itemDataJson = s"$itemSubdir/item-data.json" + private val argsMetaJson = s"$itemSubdir/given-data.json" + private val pdfDir = s"$itemSubdir/pdfs" + private val originalDir = s"$itemSubdir/originals" + private val originalMetaJson = s"$itemSubdir/source-files.json" + private val pdfMetaJson = s"$itemSubdir/pdf-files.json" + + // This environment can be used by the addon to access data of the current task + private val itemEnv = Map( + "ITEM_DIR" -> itemSubdir, + "ITEM_DATA_JSON" -> itemDataJson, + "ITEM_ARGS_JSON" -> argsMetaJson, + "ITEM_PDF_DIR" -> pdfDir, + "ITEM_ORIGINAL_DIR" -> originalDir, + "ITEM_ORIGINAL_JSON" -> originalMetaJson, + "ITEM_PDF_JSON" -> pdfMetaJson + ) + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ItemData] = + addonResult(ops, store, trigger, addonTaskIds)(collective, data, maybeMeta).as( + data + ) + + def addonResult[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ExecResult] = + Task { ctx => + ops.execAll(collective, Set(trigger), addonTaskIds, ctx.logger.some)( + Middleware.prepare(Kleisli(prepareItemData(ctx.logger, store, data, maybeMeta))) + ) + } + + def prepareItemData[F[_]: Async]( + logger: Logger[F], + store: Store[F], + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + )( + input: InputEnv + ): F[InputEnv] = + for { + _ <- logger.debug(s"Preparing item data '${data.item.name}' for addon") + wd = input.baseDir + itemMetaFile = wd / itemDataJson + argsMetaFile = wd / argsMetaJson + pdfs = wd / pdfDir + originals = wd / originalDir + srcJson = wd / originalMetaJson + pdfJson = wd / pdfMetaJson + + _ <- List(wd / itemSubdir, pdfs, originals).traverse(Files[F].createDirectories) + + _ <- logger.debug("Writing collected item data…") + _ <- itemMetaFile.writeJson(data) + + _ <- OptionT + .fromOption[F](maybeMeta) + .semiflatMap { meta => + logger.debug("Writing context meta data…") *> + argsMetaFile.writeJson(meta) + } + .value + + _ <- logger.debug("Storing all attachments…") + _ <- data.attachments + .flatMap(a => + Vector( + pdfs / a.id.id -> a.fileId, + originals / a.id.id -> data.originFile(a.id) + ) + ) + .traverse_ { case (out, key) => + logger.debug(s"Storing attachment $out") *> + store.fileRepo + .getBytes(key) + .through(Files[F].writeAll(out)) + .compile + .drain + } + + _ <- logger.debug("Storing file metadata") + srcMeta <- store.transact(QAttachment.attachmentSourceFile(data.item.id)) + pdfMeta <- store.transact(QAttachment.attachmentFile(data.item.id)) + _ <- srcJson.writeJson(srcMeta) + _ <- pdfJson.writeJson(pdfMeta) + } yield input.addEnv(itemEnv) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala new file mode 100644 index 00000000..32f8bc7b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.{ItemAddonTaskArgs, MetaProposalList} +import docspell.joex.process.ItemData +import docspell.scheduler.{PermanentError, Task} +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +object ItemAddonTask extends AddonTaskExtension { + type Args = ItemAddonTaskArgs + val name = ItemAddonTaskArgs.taskName + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + + def apply[F[_]: Async](ops: AddonOps[F], store: Store[F]): Task[F, Args, Result] = + Task { ctx => + (for { + item <- OptionT( + store.transact( + RItem.findByIdAndCollective(ctx.args.itemId, ctx.args.collective) + ) + ) + data <- OptionT.liftF(makeItemData(store, item)) + inner = GenericItemAddonTask.addonResult( + ops, + store, + AddonTriggerType.ExistingItem, + ctx.args.addonRunConfigs + )(ctx.args.collective, data, None) + execResult <- OptionT.liftF(inner.run(ctx.unit)) + _ <- OptionT.liftF(execResult.combined.raiseErrorIfNeeded[F]) + } yield Result( + execResult.combined.addonResult, + execResult.runConfigs.flatMap(_.refs).map(_.archive.nameAndVersion).distinct + )).getOrElseF( + Async[F].raiseError( + PermanentError( + new NoSuchElementException(s"Item not found for id: ${ctx.args.itemId.id}!") + ) + ) + ) + } + + def makeItemData[F[_]: Async](store: Store[F], item: RItem): F[ItemData] = + for { + attachs <- store.transact(RAttachment.findByItem(item.id)) + rmeta <- store.transact(QAttachment.getAttachmentMetaOfItem(item.id)) + rsource <- store.transact(RAttachmentSource.findByItem(item.id)) + proposals <- store.transact(QAttachment.getMetaProposals(item.id, item.cid)) + tags <- store.transact(RTag.findByItem(item.id)) + } yield ItemData( + item = item, + attachments = attachs, + metas = rmeta, + dateLabels = Vector.empty, + originFile = rsource.map(r => (r.id, r.fileId)).toMap, + givenMeta = proposals, + tags = tags.map(_.name).toList, + classifyProposals = MetaProposalList.empty, + classifyTags = Nil + ) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/Result.scala b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala new file mode 100644 index 00000000..fdd86ba2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import docspell.addons.AddonResult +import docspell.scheduler.JobTaskResultEncoder + +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder + +case class Result(addonResult: AddonResult, addons: List[String]) + +object Result { + val empty: Result = + Result(AddonResult.empty, Nil) + + implicit val jsonEncoder: Encoder[Result] = + deriveEncoder + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[Result] = + JobTaskResultEncoder.fromJson[Result].withMessage { result => + result.addonResult match { + case AddonResult.Success(_) => + s"Executed ${result.addons.size} addon(s) successfully." + + case AddonResult.ExecutionError(rc) => + s"Addon execution finished with non-zero return code: $rc" + + case AddonResult.ExecutionFailed(ex) => + s"Addon execution failed: ${ex.getMessage}" + + case AddonResult.DecodingError(msg) => + s"Addon output failed to read: $msg" + } + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala new file mode 100644 index 00000000..7f108223 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.backend.ops.OAddons +import docspell.common.{Ident, ScheduledAddonTaskArgs} +import docspell.scheduler.Task + +object ScheduledAddonTask extends AddonTaskExtension with LoggerExtension { + type Args = ScheduledAddonTaskArgs + + val name: Ident = OAddons.scheduledAddonTaskName + + def apply[F[_]: Async](ops: AddonOps[F]): Task[F, Args, Result] = + Task { ctx => + for { + execRes <- ops.execById(ctx.args.collective, ctx.args.addonTaskId, ctx.logger)( + Middleware.identity[F] + ) + _ <- execRes.result.combineAll.raiseErrorIfNeeded[F] + } yield Result( + execRes.result.combineAll.addonResult, + execRes.runConfigs.flatMap(_.refs.map(_.archive.nameAndVersion)) + ) + } + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala index 4e6d9f52..ea0500c6 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.split.TextSplitter import docspell.common._ +import docspell.common.util.File import docspell.store.queries.QCollective import io.circe.generic.semiauto._ diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 2f46234e..8d3f3562 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.store.Store import docspell.store.queries.QCollective import docspell.store.records.REquipment diff --git a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala index 1d61f4fd..24244d6b 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala @@ -14,6 +14,7 @@ import fs2.io.file.Path import docspell.analysis.classifier.{ClassifierModel, TextClassifier} import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger import docspell.store.Store import docspell.store.records.RClassifierModel diff --git a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala index 6fc5f634..0470fcdc 100644 --- a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala @@ -109,7 +109,7 @@ object MultiUploadArchiveTask { )(file: ProcessItemArgs.File): Stream[F, ProcessItemArgs] = store.fileRepo .getBytes(file.fileMetaId) - .through(Zip.unzipP[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) + .through(Zip.unzip[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) .flatMap { entry => val hint = MimeTypeHint(entry.name.some, entry.mime.asString.some) entry.data diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index 31c1e007..98a2923b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -69,7 +69,7 @@ object AttachmentPreview { } case mt => - ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot get page count.") *> + ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot create preview.") *> (None: Option[RAttachmentPreview]).pure[F] } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index 5e2d86b0..27cbe414 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -146,7 +146,7 @@ object ExtractArchive { val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all) ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *> zipData - .through(Zip.unzipP[F](8192, glob)) + .through(Zip.unzip[F](8192, glob)) .zipWithIndex .flatMap(handleEntry(ctx, store, ra, pos, archive, None)) .foldMonoid diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index c96c0189..94a6c07f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -26,7 +26,8 @@ import io.circe.{Encoder, Json} * @param dateLabels * a separate list of found dates * @param originFile - * a mapping from an attachment id to a filemeta-id containng the source or origin file + * a mapping from an attachment id to a filemeta-id containing the source or origin + * file * @param givenMeta * meta data to this item that was not "guessed" from an attachment but given and thus * is always correct @@ -49,7 +50,7 @@ case class ItemData( ) { /** sort by weight; order of equal weights is not important, just choose one others are - * then suggestions doc-date is only set when given explicitely, not from "guessing" + * then suggestions doc-date is only set when given explicitly, not from "guessing" */ def finalProposals: MetaProposalList = MetaProposalList @@ -98,7 +99,7 @@ object ItemData { dates.map(dl => dl.label.copy(label = dl.date.toString)) } - // Used to encode the result passed to the job-done event + // Used to encode the result passed to the job-done event and to supply to addons implicit val jsonEncoder: Encoder[ItemData] = Encoder.instance { data => val metaMap = data.metas.groupMap(_.id)(identity) @@ -108,10 +109,12 @@ object ItemData { "collective" -> data.item.cid.asJson, "source" -> data.item.source.asJson, "attachments" -> data.attachments + .sortBy(_.position) .map(a => Json.obj( "id" -> a.id.asJson, "name" -> a.name.asJson, + "position" -> a.position.asJson, "content" -> metaMap.get(a.id).flatMap(_.head.content).asJson, "language" -> metaMap.get(a.id).flatMap(_.head.language).asJson, "pages" -> metaMap.get(a.id).flatMap(_.head.pages).asJson @@ -123,6 +126,18 @@ object ItemData { "assumedCorrOrg" -> data.finalProposals .find(MetaProposalType.CorrOrg) .map(_.values.head.ref) + .asJson, + "assumedCorrPerson" -> data.finalProposals + .find(MetaProposalType.CorrPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcPerson" -> data.finalProposals + .find(MetaProposalType.ConcPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcEquip" -> data.finalProposals + .find(MetaProposalType.ConcEquip) + .map(_.values.head.ref) .asJson ) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 8d59a969..b5fc216e 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.Stream import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} import docspell.ftsclient.FtsClient @@ -41,7 +42,8 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] ): Task[F, Args, Option[ItemData]] = logBeginning[F].flatMap(_ => DuplicateCheck[F](store) @@ -52,7 +54,17 @@ object ItemHandler { CreateItem[F](store).contramap(_ => args.pure[F]) create .flatMap(itemStateTask(store, ItemState.Processing)) - .flatMap(safeProcess[F](cfg, store, itemOps, fts, analyser, regexNer)) + .flatMap( + safeProcess[F]( + cfg, + store, + itemOps, + fts, + analyser, + regexNer, + addons + ) + ) .map(_.some) } ) @@ -76,11 +88,14 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data).attempt + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -91,7 +106,9 @@ object ItemHandler { .andThen(_ => Sync[F].raiseError(ex)) } case false => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data) + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ) .flatMap(itemStateTask(store, ItemState.Created)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 6087b37f..836a8062 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -9,7 +9,9 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs import docspell.ftsclient.FtsClient @@ -26,6 +28,7 @@ object ProcessItem { fts: FtsClient[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = ExtractArchive(store)(item) @@ -35,6 +38,7 @@ object ProcessItem { .flatMap(SetGivenData.onlyNew[F](itemOps)) .flatMap(Task.setProgress(99)) .flatMap(RemoveEmptyItem(itemOps)) + .flatMap(RunAddons(addonOps, store, AddonTriggerType.FinalProcessItem)) def processAttachments[F[_]: Async]( cfg: Config, diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 1863d2ef..6890e37d 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -10,7 +10,9 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common._ import docspell.ftsclient.FtsClient @@ -34,13 +36,24 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] ): Task[F, Args, Unit] = Task .log[F, Args](_.info("===== Start reprocessing ======")) .flatMap(_ => loadItem[F](store) - .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer, store)) + .flatMap( + safeProcess[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store + ) + ) .map(_ => ()) ) @@ -99,6 +112,7 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F], data: ItemData ): Task[F, Args, ItemData] = { @@ -129,6 +143,7 @@ object ReProcessItem { .processAttachments[F](cfg, fts, analyser, regexNer, store)(data) .flatMap(LinkProposal[F](store)) .flatMap(SetGivenData[F](itemOps)) + .flatMap(RunAddons[F](addonOps, store, AddonTriggerType.FinalReprocessItem)) .contramap[Args](convertArgs(lang)) } } @@ -153,11 +168,21 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data).attempt + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -167,7 +192,16 @@ object ReProcessItem { ).andThen(_ => Sync[F].raiseError(ex)) } case false => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data) + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ) } private def logWarn[F[_]](msg: => String): Task[F, Args, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala new file mode 100644 index 00000000..b564d8b4 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.process + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.ProcessItemArgs +import docspell.joex.addon.GenericItemAddonTask +import docspell.scheduler.Task +import docspell.store.Store + +/** Run registered addons in the context of item processing. The addon has access to the + * current item data and can apply custom processing logic. + */ +object RunAddons { + type Args = ProcessItemArgs + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType + )( + data: ItemData + ): Task[F, Args, ItemData] = + if (data.item.state.isInvalid && data.attachments.isEmpty) { + Task.pure(data) + } else + Task { ctx => + val inner = GenericItemAddonTask(ops, store, trigger, Set.empty)( + ctx.args.meta.collective, + data, + ctx.args.meta.some + ) + inner.run(ctx.unit) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala index 0734d294..7fd35abe 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -44,7 +44,7 @@ object SetGivenData { for { _ <- ctx.logger.info("Starting setting given data") _ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'") - e <- ops.setFolder(itemId, folderId, collective).attempt + e <- ops.setFolder(itemId, folderId.map(_.id), collective).attempt _ <- e.fold( ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"), res => diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala index 6810f3d9..e02ff31d 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Duration, Ident, Timestamp} -import docspell.joex.JoexApp +import docspell.joex.{Config, JoexApp} import docspell.joexapi.model._ import docspell.store.records.RJobLog @@ -20,7 +20,7 @@ import org.http4s.dsl.Http4sDsl object JoexRoutes { - def apply[F[_]: Async](app: JoexApp[F]): HttpRoutes[F] = { + def apply[F[_]: Async](cfg: Config, app: JoexApp[F]): HttpRoutes[F] = { val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { @@ -64,6 +64,11 @@ object JoexRoutes { BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found") ) } yield resp + + case GET -> Root / "addon" / "config" => + val data = + AddonSupport(cfg.appId, cfg.addons.executorConfig.runner) + Ok(data) } } diff --git a/modules/joexapi/src/main/resources/joex-openapi.yml b/modules/joexapi/src/main/resources/joex-openapi.yml index 27bb62eb..c0e1f55a 100644 --- a/modules/joexapi/src/main/resources/joex-openapi.yml +++ b/modules/joexapi/src/main/resources/joex-openapi.yml @@ -122,8 +122,45 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /addon/config: + get: + operationId: "v1-addon-config-get" + tags: [ Addons ] + summary: What is supported running addons + description: | + Return what this joex supports when executing addons. + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonSupport" + components: schemas: + AddonSupport: + description: | + How this joex supports executing addons. + required: + - nodeId + - runners + properties: + nodeId: + type: string + format: ident + runners: + type: array + items: + type: string + format: addon-runner-type + enum: + - nix-flake + - docker + - trivial + JobAndLog: description: | Some more details about the job. diff --git a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala index c623eba0..b9b5b5ef 100644 --- a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala +++ b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Ident, LenientUri} -import docspell.joexapi.model.BasicResult +import docspell.joexapi.model.{AddonSupport, BasicResult} import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.circe.CirceEntityDecoder @@ -25,6 +25,7 @@ trait JoexClient[F[_]] { def cancelJob(base: LenientUri, job: Ident): F[BasicResult] + def getAddonSupport(base: LenientUri): F[AddonSupport] } object JoexClient { @@ -33,6 +34,13 @@ object JoexClient { new JoexClient[F] with CirceEntityDecoder { private[this] val logger = docspell.logging.getLogger[F] + def getAddonSupport(base: LenientUri): F[AddonSupport] = { + val getUrl = base / "api" / "v1" / "addon" / "config" + val req = Request[F](Method.GET, uri(getUrl)) + logger.debug(s"Getting addon support") *> + client.expect[AddonSupport](req) + } + def notifyJoex(base: LenientUri): F[BasicResult] = { val notifyUrl = base / "api" / "v1" / "notify" val req = Request[F](Method.POST, uri(notifyUrl)) diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala index 83f320bb..f5aaee15 100644 --- a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala +++ b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala @@ -143,7 +143,9 @@ object Event { } - /** Some generic list of items, chosen by a user. */ + /** Some generic list of items, chosen by a user. This is use to notify about periodic + * search results. + */ final case class ItemSelection( account: AccountId, items: Nel[Ident], diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 7a509842..ab9cc8a9 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -5790,9 +5790,388 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /sec/addon/archive: + get: + operationId: "sec-addon-archive-get" + tags: [Addons] + summary: Get all registered addons + description: | + Returns a list of all registered addons. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonList" + post: + operationId: "sec-addon-post" + tags: [ Addons ] + summary: Install a new addon + description: | + Given an URL to an addon (which is a zip file containing a + `docspell-meta.yaml` or json descriptor), the addon is + downloaded and installed in docspell. + + By default this happens asynchronously and the response only + indicates that installing has been submitted. The result will + be transfered over the websocket channel. With query parameter + `sync` installing happens synchronously and it may take a + while to complete (if successful, the addon id is returned). + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRegister" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + /sec/addon/archive/{id}: + parameters: + - $ref: "#/components/parameters/id" + delete: + operationId: "sec-addon-archive-delete" + tags: [Addons] + summary: Deletes the addon and removes it from all addon run configs + description: | + Deletes the addon from the database and also removes it from + all run configurations where it might be referenced. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + operationId: "sec-addon-archive-put" + tags: [ Addons ] + summary: Update an addon from its url + description: | + Addons are urls to zip files. This call reads the url again + and updates the contents in docspell for this addon. + + By default this happens asynchronously and the response only + indicates that updating has been submitted. The result will be + transfered over the websocket channel. With query parameter + `sync` updating happens synchronously and it may take a while + to complete. + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run-config: + get: + operationId: "sec-addon-run-config-get" + tags: [Addons] + summary: Get all addon run configs + description: | + Returns a list of addon run configs. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfigList" + post: + operationId: "sec-addon-run-config-post" + tags: [ Addons ] + summary: Adds a new addon run config + description: | + Adds a new set of configured addons, creating a run + configuration. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + + /sec/addon/run-config/{id}: + parameters: + - $ref: "#/components/parameters/id" + put: + operationId: "sec-addon-run-config-id-put" + tags: [ Addons ] + summary: Updates an addon run config + description: | + Updates an existing addon run configuration. The id is taken + from the URL and any given id in the request body is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + delete: + operationId: "sec-addonrunconfig-delete" + tags: [Addons] + summary: Deletes the addon run config given its id + description: | + Deletes the addon run configuration. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run/existingitem: + post: + operationId: "sec-addon-run-existing-item" + tags: [Addons] + summary: Submits a task running addons for an item + description: | + Submits a background task that executes the specified (or all) + addons configured to use for an existing item. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunExistingItem" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" components: schemas: + AddonRunExistingItem: + description: | + Data to run addons for an existing item. + required: + - itemId + properties: + itemId: + type: string + format: ident + additionalItems: + type: array + items: + type: string + format: ident + description: | + Additional items to run addons on. There will be one job + submitted per item. + addonRunConfigIds: + type: array + items: + type: string + format: ident + description: | + If non empty, only select these addon run configs. + Otherwise all configured to be run for existing items are + executed. + + AddonRef: + description: | + A reference to an addon (archive) with additional name and + version and its arguments. When used for adding addon run + configs, name and version are ignored. + required: + - addonId + - name + - version + - args + properties: + addonId: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + args: + type: string + + AddonRunConfig: + description: | + A set of configured addons that are run on certain points + defined by the `trigger` property. + required: + - id + - name + - enabled + - trigger + - addons + properties: + id: + type: string + format: ident + name: + type: string + enabled: + type: boolean + userId: + type: string + format: ident + description: | + An addon can be run on behalf of a user. If not given, no + authentication token is generated into the environment of + the addon. The user can be given as user_id or by its + login name. + schedule: + type: string + format: calevent + description: | + A schedule must be supplied when a trigger type of + 'scheduled' is defined. + trigger: + description: | + Defines when this task is executed. There must be at least + one element. Possible values: + + * process-item: After an item has been processed + * reprocess-item: After an item has been re-processed + * scheduled: Executed periodically based on a schedule, + which must be defined then + type: array + items: + type: string + format: addon-trigger-type + enum: + - process-item + - reprocess-item + - scheduled + addons: + type: array + items: + $ref: "#/components/schemas/AddonRef" + + AddonRunConfigList: + description: | + A list of addon run configurations. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/AddonRunConfig" + + AddonRegister: + description: | + Data to register addons + required: + - url + properties: + url: + type: string + format: uri + + Addon: + description: | + An registered addon. + required: + - id + - name + - version + - created + properties: + id: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + url: + type: string + format: uri + created: + type: integer + format: date-time + + AddonList: + description: | + A list of addons + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Addon" + DownloadAllSummary: description: | Information about a ZIP download. diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 508bb277..2c6d4523 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -462,5 +462,39 @@ docspell.server { } } } + + addons = { + enabled = false + + # Whether installing addons requiring network should be allowed + # or not. + allow-impure = true + + # Define patterns of urls that are allowed to install addons + # from. + # + # A pattern is compared against an URL by comparing three parts + # of an URL via globs: scheme, host and path. + # + # You can use '*' (0 or more) and '?' (one) as wildcards in each + # part. For example: + # + # https://*.mydomain.com/projects/* + # *s://gitea.mydomain/* + # + # A hostname is separated by dots and the path by a slash. A '*' + # in a pattern means to match one or more characters. The path + # pattern is always matching the given prefix. So /a/b/* matches + # /a/b/c and /a/b/c/d and all other sub-paths. + # + # Multiple patterns can be defined va a comma separated string + # or as an array. An empty string matches no URL, while the + # special pattern '*' all by itself means to match every URL. + allowed-urls = "*" + + # Same as `allowed-urls` but a match here means do deny addons + # from this url. + denied-urls = "" + } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 5d2b26a6..45f58c2e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -152,7 +152,9 @@ final class RestAppImpl[F[_]: Async]( "clientSettings" -> ClientSettingsRoutes(backend, token), "notification" -> NotificationRoutes(config, backend, token), "querybookmark" -> BookmarkRoutes(backend, token), - "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token) + "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token), + "addonrunconfig" -> AddonRunConfigRoutes(backend, token), + "addon" -> AddonRoutes(config, wsTopic, backend, token) ) } @@ -181,7 +183,16 @@ object RestAppImpl { .withEventSink(notificationMod) .build backend <- BackendApp - .create[F](store, javaEmil, ftsClient, pubSubT, schedulerMod, notificationMod) + .create[F]( + cfg.backend, + store, + javaEmil, + httpClient, + ftsClient, + pubSubT, + schedulerMod, + notificationMod + ) app = new RestAppImpl[F]( cfg, diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 1a8fd6c4..e4c1a5bc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -14,6 +14,7 @@ import fs2.Stream import fs2.concurrent.Topic import docspell.backend.msg.Topics +import docspell.backend.ops.ONode import docspell.common._ import docspell.pubsub.naive.NaivePubSub import docspell.restserver.http4s.InternalHeader @@ -91,6 +92,15 @@ object RestServer { store, httpClient )(Topics.all.map(_.topic)) + + nodes <- ONode(store) + _ <- nodes.withRegistered( + cfg.appId, + NodeType.Restserver, + cfg.baseUrl, + cfg.auth.serverSecret.some + ) + restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic) } yield (restApp, pubSub, setting) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala new file mode 100644 index 00000000..dd2351b8 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.conv + +import cats.syntax.all._ + +import docspell.addons.AddonMeta +import docspell.backend.ops.AddonValidationError +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.Ident +import docspell.restserver.ws.{OutputEvent, OutputEventEncoder} +import docspell.store.records.RAddonArchive + +trait AddonValidationSupport { + + def validationErrorToMessage(e: AddonValidationError): String = + e match { + case AddonValidationError.AddonNotFound => + "Addon not found." + + case AddonValidationError.AddonExists(msg, _) => + msg + + case AddonValidationError.NotAnAddon(ex) => + s"The url doesn't seem to be an addon: ${ex.getMessage}" + + case AddonValidationError.InvalidAddon(msg) => + s"The addon is not valid: $msg" + + case AddonValidationError.AddonUnsupported(msg, _) => + msg + + case AddonValidationError.AddonsDisabled => + "Addons are disabled in the config file." + + case AddonValidationError.UrlUntrusted(_) => + "This url doesn't belong to te set of trusted urls defined in the config file" + + case AddonValidationError.DownloadFailed(ex) => + s"Downloading the addon failed: ${ex.getMessage}" + + case AddonValidationError.ImpureAddonsDisabled => + s"Installing impure addons is disabled." + + case AddonValidationError.RefreshLocalAddon => + "Refreshing a local addon doesn't work." + } + + def addonResultOutputEventEncoder( + collective: Ident + ): OutputEventEncoder[AddonValidationResult[(RAddonArchive, AddonMeta)]] = + OutputEventEncoder.instance { + case Right((archive, _)) => + OutputEvent.AddonInstalled( + collective, + "Addon installed", + None, + archive.id.some, + archive.originalUrl + ) + + case Left(error) => + val msg = validationErrorToMessage(error) + OutputEvent.AddonInstalled(collective, msg, error.some, None, None) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala index aead3504..6b17d49e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala @@ -47,4 +47,8 @@ object Responses { def notFoundRoute[F[_]: Sync]: HttpRoutes[F] = HttpRoutes(_ => OptionT.pure(Response.notFound[F])) + def notFoundRoute[F[_]: Sync, A](body: A)(implicit + entityEncoder: EntityEncoder[F, A] + ): HttpRoutes[F] = + HttpRoutes(_ => OptionT.pure(Response.notFound[F].withEntity(body))) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala new file mode 100644 index 00000000..ceeabadc --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.http4s + +import cats.effect._ + +import docspell.joexapi.model.BasicResult + +import org.http4s.Response +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +trait ThrowableResponseMapper { + + implicit class EitherThrowableOps[A](self: Either[Throwable, A]) { + def rightAs[F[_]: Sync](f: A => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], f) + + def rightAs_[F[_]: Sync](r: => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], _ => r) + } +} + +object ThrowableResponseMapper { + def toResponse[F[_]: Sync](ex: Throwable): F[Response[F]] = + new Mapper[F].toResponse(ex) + + private class Mapper[F[_]: Sync] extends Http4sDsl[F] { + def toResponse(ex: Throwable): F[Response[F]] = + ex match { + case _: IllegalArgumentException => + BadRequest(BasicResult(false, ex.getMessage)) + + case _ => + InternalServerError(BasicResult(false, ex.getMessage)) + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala new file mode 100644 index 00000000..72c9b808 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.addons.AddonMeta +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.conv.AddonValidationSupport +import docspell.restserver.ws.{Background, OutputEvent} +import docspell.store.records.RAddonArchive + +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl +import org.http4s.dsl.impl.FlagQueryParamMatcher +import org.http4s.{HttpRoutes, Response} + +object AddonArchiveRoutes extends AddonValidationSupport { + + def apply[F[_]: Async]( + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + implicit val wsOutputEnc = addonResultOutputEventEncoder(token.account.collective) + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddons(token.account.collective) + resp <- Ok( + AddonList( + all.map(r => + Addon(r.id, r.name, r.version, r.description, r.originalUrl, r.created) + ) + ) + ) + } yield resp + + case req @ POST -> Root :? Sync(sync) => + def create(r: Option[RAddonArchive]) = + IdResult( + true, + r.fold("Addon submitted for installation")(r => + s"Addon installed: ${r.id.id}" + ), + r.map(_.id).getOrElse(Ident.unsafe("")) + ) + + for { + input <- req.as[AddonRegister] + install = backend.addons.registerAddon( + token.account.collective, + input.url, + None + ) + resp <- + if (sync) + install.flatMap( + _.fold(convertAddonValidationError[F], r => Ok(create(r._1.some))) + ) + else Background(wsTopic)(install).flatMap(_ => Ok(create(None))) + } yield resp + + case PUT -> Root / Ident(id) :? Sync(sync) => + def create(r: Option[AddonMeta]) = + BasicResult( + true, + r.fold("Addon updated in background")(m => + s"Addon updated: ${m.nameAndVersion}" + ) + ) + val update = backend.addons.refreshAddon(token.account.collective, id) + for { + resp <- + if (sync) + update.flatMap( + _.fold( + convertAddonValidationError[F], + r => Ok(create(r._2.some)) + ) + ) + else Background(wsTopic)(update).flatMap(_ => Ok(create(None))) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddon(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon deleted")) + else NotFound(BasicResult(false, "Addon not found")) + } yield resp + } + } + + def convertAddonValidationError[F[_]: Async]( + e: AddonValidationError + ): F[Response[F]] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + + def failWith(msg: String): F[Response[F]] = + Ok(IdResult(false, msg, Ident.unsafe(""))) + + e match { + case AddonValidationError.AddonNotFound => + NotFound(BasicResult(false, "Addon not found.")) + + case _ => + failWith(validationErrorToMessage(e)) + } + } + + object Sync extends FlagQueryParamMatcher("sync") +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala new file mode 100644 index 00000000..192f6ccb --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect.Async +import fs2.concurrent.Topic + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model.BasicResult +import docspell.restserver.Config +import docspell.restserver.http4s.Responses +import docspell.restserver.ws.OutputEvent + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.server.Router + +object AddonRoutes { + + def apply[F[_]: Async]( + cfg: Config, + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = + if (cfg.backend.addons.enabled) + Router( + "archive" -> AddonArchiveRoutes(wsTopic, backend, token), + "run-config" -> AddonRunConfigRoutes(backend, token), + "run" -> AddonRunRoutes(backend, token) + ) + else + Responses.notFoundRoute(BasicResult(false, "Addons disabled")) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala new file mode 100644 index 00000000..d146eef1 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.OAddons +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunConfigRoutes { + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddonRunConfigs(token.account.collective) + resp <- Ok(AddonRunConfigList(all.map(convertInfoTask))) + } yield resp + + case req @ POST -> Root => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(Ident.unsafe(""), input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config added", id)) + ) + } yield resp + + case req @ PUT -> Root / Ident(id) => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(id, input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config updated", id)) + ) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddonRunConfig(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon task deleted")) + else NotFound(BasicResult(false, "Addon task not found")) + } yield resp + } + } + + def convertInsertTask( + id: Ident, + t: AddonRunConfig + ): Either[String, OAddons.AddonRunInsert] = + for { + tr <- NonEmptyList + .fromList(t.trigger) + .toRight("At least one trigger is required") + ta <- NonEmptyList + .fromList(t.addons) + .toRight("At least one addon is required") + res = OAddons.AddonRunInsert( + id, + t.name, + t.enabled, + t.userId, + t.schedule, + tr, + ta.map(e => OAddons.AddonArgs(e.addonId, e.args)) + ) + } yield res + + def convertInfoTask(t: OAddons.AddonRunInfo): AddonRunConfig = + AddonRunConfig( + id = t.id, + name = t.name, + enabled = t.enabled, + userId = t.userId, + schedule = t.schedule, + trigger = t.triggered, + addons = t.addons.map { case (ra, raa) => + AddonRef(raa.addonId, ra.name, ra.version, ra.description, raa.args) + } + ) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala new file mode 100644 index 00000000..3b167b2d --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunRoutes { + + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { case req @ POST -> Root / "existingitem" => + for { + input <- req.as[AddonRunExistingItem] + _ <- backend.addons.runAddonForItem( + token.account, + NonEmptyList(input.itemId, input.additionalItems), + input.addonRunConfigIds.toSet + ) + resp <- Ok(BasicResult(true, "Job for running addons submitted.")) + } yield resp + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala index a713f160..1a8ebd6a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala @@ -117,7 +117,11 @@ object ItemMultiRoutes extends NonEmptyListSupport with MultiIdSupport { for { json <- req.as[ItemsAndRef] items <- requireNonEmpty(json.items) - res <- backend.item.setFolderMultiple(items, json.ref, user.account.collective) + res <- backend.item.setFolderMultiple( + items, + json.ref.map(_.id), + user.account.collective + ) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index d830fc6c..bcabb4c7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -218,7 +218,7 @@ object ItemRoutes { case req @ PUT -> Root / Ident(id) / "folder" => for { idref <- req.as[OptionalId] - res <- backend.item.setFolder(id, idref.id, user.account.collective) + res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala index de54198c..5faf9f76 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala @@ -29,7 +29,8 @@ case class Flags( downloadAllMaxFiles: Int, downloadAllMaxSize: ByteSize, uiVersion: Int, - openIdAuth: List[Flags.OpenIdAuth] + openIdAuth: List[Flags.OpenIdAuth], + addonsEnabled: Boolean ) object Flags { @@ -47,7 +48,8 @@ object Flags { cfg.downloadAll.maxFiles, cfg.downloadAll.maxSize, uiVersion, - cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)) + cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)), + cfg.backend.addons.enabled ) final case class OpenIdAuth(provider: Ident, name: String) diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala new file mode 100644 index 00000000..b0870fc5 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.logging.Logger + +/** Asynchronous operations that run on the rest-server can communicate their results via + * websocket. + */ +object Background { + // TODO avoid resubmitting same stuff + + def apply[F[_]: Async, A]( + wsTopic: Topic[F, OutputEvent], + logger: Option[Logger[F]] = None + )(run: F[A])(implicit enc: OutputEventEncoder[A]): F[Unit] = { + val log = logger.getOrElse(docspell.logging.getLogger[F]) + Async[F] + .background(run) + .use( + _.flatMap( + _.fold( + log.warn("The background operation has been cancelled!"), + ex => log.error(ex)("Error running background operation!"), + event => + event + .map(enc.encode) + .flatTap(ev => log.info(s"Sending response from async operation: $ev")) + .flatMap(wsTopic.publish1) + .void + ) + ) + ) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala index c7e57be4..5ac68c48 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala @@ -7,6 +7,7 @@ package docspell.restserver.ws import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError import docspell.common._ import io.circe._ @@ -55,6 +56,29 @@ object OutputEvent { Msg("jobs-waiting", count).asJson } + final case class AddonInstalled( + collective: Ident, + message: String, + error: Option[AddonValidationError], + addonId: Option[Ident], + originalUrl: Option[LenientUri] + ) extends OutputEvent { + def forCollective(token: AuthToken) = + token.account.collective == collective + + override def asJson = + Msg( + "addon-installed", + Map( + "success" -> error.isEmpty.asJson, + "error" -> error.asJson, + "addonId" -> addonId.asJson, + "addonUrl" -> originalUrl.asJson, + "message" -> message.asJson + ) + ).asJson + } + private case class Msg[A](tag: String, content: A) private object Msg { implicit def jsonEncoder[A: Encoder]: Encoder[Msg[A]] = diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala new file mode 100644 index 00000000..4414dfad --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +trait OutputEventEncoder[A] { + def encode(a: A): OutputEvent +} + +object OutputEventEncoder { + def apply[A](implicit e: OutputEventEncoder[A]): OutputEventEncoder[A] = e + + def instance[A](f: A => OutputEvent): OutputEventEncoder[A] = + (a: A) => f(a) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala index 0594dcf8..4ca51dc9 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -6,6 +6,8 @@ package docspell.scheduler +import cats.effect.Sync + import docspell.common._ import docspell.logging.Logger @@ -25,4 +27,8 @@ trait Context[F[_], A] { self => def map[C](f: A => C): Context[F, C] + def unit: Context[F, Unit] = + map(_ => ()) + + def loadJob(implicit F: Sync[F]): F[Job[String]] } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala new file mode 100644 index 00000000..7a921769 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +/** Special "marker" exception to indicate errors in tasks, that should NOT be retried. */ +final class PermanentError(cause: Throwable) extends RuntimeException(cause) { + override def fillInStackTrace() = this +} + +object PermanentError { + def apply(cause: Throwable): PermanentError = + new PermanentError(cause) + + def isPermanent(ex: Throwable): Boolean = + unapply(ex).isDefined + + def unapply(ex: Throwable): Option[Throwable] = + ex match { + case p: PermanentError => Some(p.getCause) + case _ => None + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala index d6868a08..b8ecd5fb 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala @@ -21,6 +21,11 @@ trait Task[F[_], A, B] { def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = Task(Task.toKleisli(this).andThen(f)) + def andThenC[C](f: (Context[F, A], B) => F[C])(implicit M: Monad[F]): Task[F, A, C] = { + val run = Task.toKleisli(this).run + Task(ctx => run(ctx).flatMap(b => f(ctx, b))) + } + def mapF[C](f: F[B] => F[C]): Task[F, A, C] = Task(Task.toKleisli(this).mapF(f)) diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala index 236c7ee6..bd3027fe 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala @@ -50,6 +50,9 @@ object UserTaskScope { def apply(collective: Ident): UserTaskScope = UserTaskScope.collective(collective) + def apply(collective: Ident, login: Option[Ident]): UserTaskScope = + login.map(AccountId(collective, _)).map(account).getOrElse(apply(collective)) + def system: UserTaskScope = collective(DocspellSystem.taskGroup) } diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala index 59016b9f..7b5a7615 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -24,6 +24,19 @@ class ContextImpl[F[_]: Functor, A]( val jobId: Ident ) extends Context[F, A] { + def loadJob(implicit F: Sync[F]): F[Job[String]] = + JobStoreImpl(store) + .findById(jobId) + .flatMap( + _.fold( + F.raiseError[Job[String]]( + new IllegalStateException(s"Job not found: ${jobId.id}") + ) + )( + F.pure + ) + ) + def setProgress(percent: Int): F[Unit] = { val pval = math.min(100, math.max(0, percent)) store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala index 5ebfd665..be263ad1 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala @@ -34,6 +34,7 @@ object LogSink { .capture("task", e.taskName) .capture("group", e.group) .capture("jobInfo", e.jobInfo) + .captureAll(e.data) e.level match { case LogLevel.Info => diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala index 50410e65..7e81cd1d 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala @@ -12,8 +12,7 @@ import cats.syntax.all._ import fs2.Stream import docspell.common.{Ident, LogLevel} -import docspell.logging -import docspell.logging.{Level, Logger} +import docspell.logging.{Level, LogEvent => DsLogEvent, Logger} /** Background tasks use this logger to emit the log events to a queue. The consumer is * [[LogSink]], which picks up log events in a separate thread. @@ -29,7 +28,7 @@ object QueueLogger { ): Logger[F] = new Logger[F] { - def log(logEvent: => logging.LogEvent) = + def log(logEvent: => DsLogEvent) = LogEvent .create[F]( jobId, diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index bc87d0fa..c114e2bc 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -267,27 +267,36 @@ final class SchedulerImpl[F[_]: Async]( .mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa) .mapF(_.attempt.flatMap { case Right(result) => - logger.info(s"Job execution successful: ${job.info}") - ctx.logger.info("Job execution successful") *> + logger.info(s"Job execution successful: ${job.info}") *> + ctx.logger.info("Job execution successful") *> (JobState.Success: JobState, result).pure[F] + + case Left(PermanentError(ex)) => + logger.warn(ex)("Task failed with permanent error") *> + ctx.logger + .warn(ex)("Task failed with permanent error!") + .as(JobState.failed -> JobTaskResult.empty) + case Left(ex) => state.get.map(_.wasCancelled(job)).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") - ctx.logger.error(ex)("Job execution failed (cancel = true)") *> + logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") *> + ctx.logger.error(ex)("Job execution failed (cancel = true)") *> (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] case false => QJob.exceedsRetries(job.id, config.retries, store).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) case false => - logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retrying later.") - .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") + .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) } } }) diff --git a/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql new file mode 100644 index 00000000..4bc97aba --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table `addon_archive`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `file_id` varchar(254) not null, + `original_url` varchar(2000), + `name` varchar(254) not null, + `version` varchar(254) not null, + `description` text, + `triggers` text not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`file_id`) references `filemeta`(`file_id`), + unique (`cid`, `original_url`), + unique (`cid`, `name`, `version`) +); + +create table `addon_run_config`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `user_id` varchar(254), + `name` varchar(254) not null, + `enabled` boolean not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`user_id`) references `user_`(`uid`) +); + +create table `addon_run_config_addon` ( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `addon_id` varchar(254) not null, + `args` text not null, + `position` int not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + foreign key (`addon_id`) references `addon_archive`(`id`) on delete cascade +); + +create table `addon_run_config_trigger`( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `triggers` varchar(254) not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + unique (`addon_run_config_id`, `triggers`) +); + +alter table `node` +add column (`server_secret` varchar(2000)); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala new file mode 100644 index 00000000..df69f421 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.file + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream + +import docspell.common.{FileKey, LenientUri, UrlReader} + +import binny.BinaryId + +object FileUrlReader { + + private val scheme: String = "docspell-file" + + def url(key: FileKey): LenientUri = + LenientUri( + scheme = Nel.of(scheme), + authority = Some(""), + path = LenientUri.NonEmptyPath( + Nel.of(key.collective.id, key.category.id.id, key.id.id) + ), + query = None, + fragment = None + ) + + def apply[F[_]: Sync](repo: FileRepository[F]): UrlReader[F] = + UrlReader.instance { url => + url.scheme.head match { + case `scheme` => + Stream + .emit(urlToFileKey(url)) + .covary[F] + .rethrow + .evalMap(key => repo.findMeta(key).map(m => (key, m))) + .flatMap { + case _ -> Some(m) => repo.getBytes(m.id) + case key -> None => + Stream.raiseError( + new NoSuchElementException( + s"File not found for url '${url.asString}' (key=$key)" + ) + ) + } + + case _ => + UrlReader.defaultReader[F].apply(url) + } + } + + private[file] def urlToFileKey(url: LenientUri): Either[Throwable, FileKey] = + BinnyUtils + .binaryIdToFileKey(BinaryId(url.host match { + case Some(h) if h.nonEmpty => s"$h${url.path.asString}" + case _ => url.path.segments.mkString("/") + })) + .leftMap(new IllegalArgumentException(_)) +} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala index 738a078b..6e919330 100644 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala @@ -9,6 +9,7 @@ package docspell.store.impl import java.time.format.DateTimeFormatter import java.time.{Instant, LocalDate} +import docspell.addons.AddonTriggerType import docspell.common._ import docspell.common.syntax.all._ import docspell.jsonminiq.JsonMiniQuery @@ -31,9 +32,9 @@ trait DoobieMeta extends EmilDoobieMeta { implicit val sqlLogging: LogHandler = LogHandler { case e @ Success(_, _, _, _) => - DoobieMeta.logger.trace("SQL " + e) + DoobieMeta.logger.trace(s"SQL: $e") case e => - DoobieMeta.logger.error(s"SQL Failure: $e") + DoobieMeta.logger.warn(s"SQL Failure: $e") } def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = @@ -41,6 +42,12 @@ trait DoobieMeta extends EmilDoobieMeta { e.apply(a).noSpaces ) + implicit val metaAddonTriggerType: Meta[AddonTriggerType] = + Meta[String].timap(AddonTriggerType.unsafeFromString)(_.name) + + implicit val metaAddonTriggerTypeSet: Meta[Set[AddonTriggerType]] = + jsonMeta[Set[AddonTriggerType]] + implicit val metaBinaryId: Meta[BinaryId] = Meta[String].timap(BinaryId.apply)(_.id) diff --git a/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala new file mode 100644 index 00000000..94e51f96 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.queries + +import docspell.common.BaseJsonCodecs._ +import docspell.common._ + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} +import scodec.bits.ByteVector + +/** Information about an attachment file (can be attachment-source or attachment) */ +case class AttachedFile( + id: Ident, + name: Option[String], + position: Int, + language: Option[Language], + mimetype: MimeType, + length: ByteSize, + checksum: ByteVector +) + +object AttachedFile { + + implicit val jsonDecoder: Decoder[AttachedFile] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachedFile] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 5cdcafb3..6e90cc86 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -22,10 +22,37 @@ import doobie._ object QAttachment { private val a = RAttachment.as("a") + private val as = RAttachmentSource.as("ats") private val item = RItem.as("i") private val am = RAttachmentMeta.as("am") private val c = RCollective.as("c") private val im = RItemProposal.as("im") + private val fm = RFileMeta.as("fm") + + def attachmentSourceFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(as.id, as.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(as, a.id === as.id) + .innerJoin(fm, fm.id === as.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] + + def attachmentFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(a.id, a.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(fm, fm.id === a.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] def deletePreview[F[_]: Sync](store: Store[F])(attachId: Ident): F[Int] = { val findPreview = @@ -163,6 +190,17 @@ object QAttachment { q.query[RAttachmentMeta].option } + def getAttachmentMetaOfItem(itemId: Ident): ConnectionIO[Vector[RAttachmentMeta]] = + Select( + select(am.all), + from(am) + .innerJoin(a, a.id === am.id), + a.itemId === itemId + ).orderBy(a.position.asc) + .build + .query[RAttachmentMeta] + .to[Vector] + case class ContentAndName( id: Ident, item: Ident, @@ -175,6 +213,7 @@ object QAttachment { def allAttachmentMetaAndName( coll: Option[Ident], itemIds: Option[Nel[Ident]], + itemStates: Nel[ItemState], chunkSize: Int ): Stream[ConnectionIO, ContentAndName] = Select( @@ -192,7 +231,7 @@ object QAttachment { .innerJoin(item, item.id === a.itemId) .innerJoin(c, c.id === item.cid) ).where( - item.state.in(ItemState.validStates) &&? + item.state.in(itemStates) &&? itemIds.map(ids => item.id.in(ids)) &&? coll.map(cid => item.cid === cid) ).build diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala new file mode 100644 index 00000000..69c6826a --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala @@ -0,0 +1,155 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.AddonTriggerType +import docspell.common.{Ident, Timestamp} +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ + +case class AddonRunConfigData( + runConfig: RAddonRunConfig, + addons: List[RAddonRunConfigAddon], + triggers: List[RAddonRunConfigTrigger] +) + +object AddonRunConfigData { + + def findAll( + cid: Ident, + enabled: Option[Boolean] = None, + trigger: Set[AddonTriggerType] = Set.empty, + configIds: Set[Ident] = Set.empty + ): ConnectionIO[List[AddonRunConfigData]] = + for { + runConfigs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + addons <- runConfigs.traverse(t => + RAddonRunConfigAddon.findByRunConfig(t.id).map(as => t.id -> as) + ) + addonMap = addons.toMap + triggers <- runConfigs.traverse(t => + RAddonRunConfigTrigger.findByRunConfig(t.id).map(ts => t.id -> ts) + ) + triggerMap = triggers.toMap + result = runConfigs.map(t => + AddonRunConfigData(t, addonMap(t.id), triggerMap(t.id)) + ) + } yield result + + /** Inserts new, creating new identifiers */ + def insert(task: AddonRunConfigData): ConnectionIO[Ident] = + for { + tid <- Ident.randomId[ConnectionIO] + now <- Timestamp.current[ConnectionIO] + tr = task.runConfig.copy(id = tid, created = now) + _ <- RAddonRunConfig.insert(tr) + _ <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigTrigger.insert) + } + _ <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield tid + + /** Updates the task, keeping its id but replacing all related objects */ + def update(task: AddonRunConfigData): ConnectionIO[Int] = + for { + n1 <- RAddonRunConfig.update(task.runConfig) + _ <- RAddonRunConfigTrigger.deleteAllForConfig(task.runConfig.id) + _ <- RAddonRunConfigAddon.deleteAllForConfig(task.runConfig.id) + tts <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigTrigger.insert) + } + tas <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield n1 + tts.sum + tas.sum + + def findEnabledRef( + cid: Ident, + taskId: Ident + ): ConnectionIO[List[(RAddonArchive, RAddonRunConfigAddon)]] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + + Select( + combineNel(select(aa.all), select(ta.all)), + from(run) + .innerJoin(ta, ta.runConfigId === run.id) + .innerJoin(aa, aa.id === ta.addonId), + run.cid === cid && run.enabled === true && run.id === taskId + ).orderBy(ta.position.asc) + .build + .query[(RAddonArchive, RAddonRunConfigAddon)] + .to[List] + } + + def findEnabledRefs( + cid: Ident, + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + ): Stream[ConnectionIO, (RAddonRunConfig, List[(RAddonArchive, String)])] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + val tt = RAddonRunConfigTrigger.as("tt") + + val taskIdFilter = NonEmptyList + .fromList(addonTaskIds.toList) + .map(nel => run.id.in(nel)) + val validTasks = TableDef("valid_task") + val validTaskId = Column[Ident]("id", validTasks) + val query = + withCte( + validTasks -> Select( + select(run.all), + from(run) + .innerJoin(tt, tt.runConfigId === run.id), + run.cid === cid && run.enabled === true && tt.trigger === trigger &&? taskIdFilter + ).distinct + )( + Select( + combineNel( + select(run.all.map(_.copy(table = validTasks))), + select(aa.all), + select(ta.args) + ), + from(validTasks) + .innerJoin(ta, ta.runConfigId === validTaskId) + .innerJoin(aa, aa.id === ta.addonId) + ).orderBy(validTaskId) + ).build + + query + .query[(RAddonRunConfig, RAddonArchive, String)] + .stream + .groupAdjacentBy(_._1.id) + .map { case (_, chunk) => + val list = chunk.toList + (list.head._1, list.map(e => (e._2, e._3))) + } + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala new file mode 100644 index 00000000..94ddb6bc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.OptionT +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ +import docspell.store.records.AddonRunConfigResolved.AddonRef + +import doobie._ +import doobie.implicits._ + +final case class AddonRunConfigResolved( + config: RAddonRunConfig, + refs: List[AddonRef], + trigger: List[RAddonRunConfigTrigger] +) {} + +object AddonRunConfigResolved { + + case class AddonRef(archive: RAddonArchive, ref: RAddonRunConfigAddon) + + def findAddonRefs(configId: Ident): ConnectionIO[List[AddonRef]] = { + val ca = RAddonRunConfigAddon.as("ca") + val aa = RAddonArchive.as("aa") + Select( + select(combineNel(aa.all, ca.all)), + from(ca) + .innerJoin(aa, aa.id === ca.addonId), + ca.runConfigId === configId + ).build.query[AddonRef].to[List] + } + + def getRefsAndTrigger( + configId: Ident + ): ConnectionIO[(List[AddonRef], List[RAddonRunConfigTrigger])] = + (findAddonRefs(configId), RAddonRunConfigTrigger.findByRunConfig(configId)).tupled + + def findById( + configId: Ident, + collective: Ident, + enabled: Option[Boolean] + ): ConnectionIO[Option[AddonRunConfigResolved]] = + (for { + cfg <- OptionT(RAddonRunConfig.findById(collective, configId)) + .filter(c => enabled.isEmpty || enabled == c.enabled.some) + (refs, tri) <- OptionT.liftF(getRefsAndTrigger(configId)) + } yield AddonRunConfigResolved(cfg, refs, tri)).value + + def findAllForCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[AddonRunConfigResolved]] = + for { + cfgs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + result <- cfgs.traverse(ac => + getRefsAndTrigger(ac.id).map { case (refs, tri) => + AddonRunConfigResolved(ac, refs, tri) + } + ) + } yield result +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala new file mode 100644 index 00000000..05e2fc15 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} +import docspell.common._ +import docspell.store.file.FileUrlReader +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class RAddonArchive( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + name: String, + version: String, + description: Option[String], + triggers: Set[AddonTriggerType], + created: Timestamp +) { + + def nameAndVersion: String = + s"${name}-${version}" + + def isUnchanged(meta: AddonMeta): Boolean = + name == meta.meta.name && + version == meta.meta.version && + description == meta.meta.description + + def isChanged(meta: AddonMeta): Boolean = + !isUnchanged(meta) + + def asArchive: AddonArchive = + AddonArchive(FileUrlReader.url(fileId), name, version) + + def update(file: FileKey, meta: AddonMeta): RAddonArchive = + copy( + fileId = file, + name = meta.meta.name, + version = meta.meta.version, + description = meta.meta.description, + triggers = meta.triggers.getOrElse(Set.empty) + ) +} + +object RAddonArchive { + case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_archive" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val fileId = Column[FileKey]("file_id", this) + val originalUrl = Column[LenientUri]("original_url", this) + val name = Column[String]("name", this) + val version = Column[String]("version", this) + val description = Column[String]("description", this) + val triggers = Column[Set[AddonTriggerType]]("triggers", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of( + id, + cid, + fileId, + originalUrl, + name, + version, + description, + triggers, + created + ) + } + + def apply( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + meta: AddonMeta, + created: Timestamp + ): RAddonArchive = + RAddonArchive( + id, + cid, + fileId, + originalUrl, + meta.meta.name, + meta.meta.version, + meta.meta.description, + meta.triggers.getOrElse(Set.empty), + created + ) + + def as(alias: String): Table = + Table(Some(alias)) + + val T = Table(None) + + def insert(r: RAddonArchive, silent: Boolean): ConnectionIO[Int] = { + val values = + sql"${r.id}, ${r.cid}, ${r.fileId}, ${r.originalUrl}, ${r.name}, ${r.version}, ${r.description}, ${r.triggers}, ${r.created}" + + if (silent) DML.insertSilent(T, T.all, values) + else DML.insert(T, T.all, values) + } + + def existsByUrl(cid: Ident, url: LenientUri): ConnectionIO[Boolean] = + Select( + select(count(T.id)), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[Int].unique.map(_ > 0) + + def findByUrl(cid: Ident, url: LenientUri): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[RAddonArchive].option + + def findByNameAndVersion( + cid: Ident, + name: String, + version: String + ): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.name === name && T.version === version + ).build.query[RAddonArchive].option + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id === id + ).build.query[RAddonArchive].option + + def findByIds(cid: Ident, ids: NonEmptyList[Ident]): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id.in(ids) + ).orderBy(T.name).build.query[RAddonArchive].to[List] + + def update(r: RAddonArchive): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id && T.cid === r.cid, + DML.set( + T.fileId.setTo(r.fileId), + T.originalUrl.setTo(r.originalUrl), + T.name.setTo(r.name), + T.version.setTo(r.version), + T.description.setTo(r.description), + T.triggers.setTo(r.triggers) + ) + ) + + def listAll(cid: Ident): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid + ).orderBy(T.name.asc).build.query[RAddonArchive].to[List] + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) + + implicit val jsonDecoder: Decoder[RAddonArchive] = deriveDecoder + implicit val jsonEncoder: Encoder[RAddonArchive] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala new file mode 100644 index 00000000..70460aaa --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfig( + id: Ident, + cid: Ident, + userId: Option[Ident], + name: String, + enabled: Boolean, + created: Timestamp +) + +object RAddonRunConfig { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val userId = Column[Ident]("user_id", this) + val name = Column[String]("name", this) + val enabled = Column[Boolean]("enabled", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, cid, userId, name, enabled, created) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfig): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.cid}, ${r.userId}, ${r.name}, ${r.enabled}, ${r.created}" + ) + + def update(r: RAddonRunConfig): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.name.setTo(r.name), + T.enabled.setTo(r.enabled), + T.userId.setTo(r.userId) + ) + ) + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonRunConfig]] = + Select(select(T.all), from(T), T.cid === cid && T.id === id).build + .query[RAddonRunConfig] + .option + + def findByCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[RAddonRunConfig]] = { + val ac = RAddonRunConfig.as("ac") + val tt = RAddonRunConfigTrigger.as("tt") + val filter = + ac.cid === cid &&? + enabled.map(e => ac.enabled === e) &&? + NonEmptyList.fromList(configIds.toList).map(ids => ac.id.in(ids)) + + val selectConfigs = + NonEmptyList.fromList(trigger.toList) match { + case Some(tri) => + Select( + select(ac.all), + from(ac).innerJoin(tt, tt.runConfigId === ac.id), + filter && tt.trigger.in(tri) + ) + case None => + Select(select(ac.all), from(ac), filter) + } + + selectConfigs.build.query[RAddonRunConfig].to[List] + } + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala new file mode 100644 index 00000000..aa95ca35 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigAddon( + id: Ident, + runConfigId: Ident, + addonId: Ident, + args: String, + position: Int +) + +object RAddonRunConfigAddon { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_addon" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val addonId = Column[Ident]("addon_id", this) + val args = Column[String]("args", this) + val position = Column[Int]("position", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, addonId, args, position) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfigAddon): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.runConfigId}, ${r.addonId}, ${r.args}, ${r.position}" + ) + + def updateArgs(addonTaskId: Ident, addonId: Ident, args: String): ConnectionIO[Int] = + DML.update( + T, + T.runConfigId === addonTaskId && T.addonId === addonId, + DML.set( + T.args.setTo(args) + ) + ) + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigAddon]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId) + .orderBy(T.position.asc) + .build + .query[RAddonRunConfigAddon] + .to[List] + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala new file mode 100644 index 00000000..fccd81f2 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigTrigger( + id: Ident, + runConfigId: Ident, + trigger: AddonTriggerType +) + +object RAddonRunConfigTrigger { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_trigger" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val trigger = Column[AddonTriggerType]("triggers", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, trigger) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) + + def insert(r: RAddonRunConfigTrigger): ConnectionIO[Int] = + DML.insert(T, T.all, sql"${r.id}, ${r.runConfigId}, ${r.trigger}") + + def insertAll( + addonTaskId: Ident, + triggers: NonEmptyList[AddonTriggerType] + ): ConnectionIO[Int] = { + val records = triggers.traverse(t => + Ident.randomId[ConnectionIO].map(id => RAddonRunConfigTrigger(id, addonTaskId, t)) + ) + val inserts = + s"INSERT INTO ${T.tableName} (id, addon_run_config_id, trigger) VALUES (?,?,?)" + records.flatMap(rs => Update[RAddonRunConfigTrigger](inserts).updateMany(rs)) + } + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigTrigger]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId).build + .query[RAddonRunConfigTrigger] + .to[List] +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 64599d6e..cb53f3d8 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -119,6 +119,9 @@ object RAttachmentMeta { def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] = DML.update(T, T.id === mid, DML.set(T.pages.setTo(pageCount))) + def updateContent(id: Ident, text: String): ConnectionIO[Int] = + DML.update(T, T.id === id, DML.set(T.content.setTo(text))) + def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index 3b9d23aa..6ca4bc8e 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -6,7 +6,8 @@ package docspell.store.records -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, OptionT} +import cats.syntax.all._ import docspell.common.{FileKey, _} import docspell.store.qb.DSL._ @@ -44,12 +45,29 @@ object RAttachmentPreview { def insert(v: RAttachmentPreview): ConnectionIO[Int] = DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") + def update(r: RAttachmentPreview): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.fileId.setTo(r.fileId), + T.name.setTo(r.name) + ) + ) + def findById(attachId: Ident): ConnectionIO[Option[RAttachmentPreview]] = run(select(T.all), from(T), T.id === attachId).query[RAttachmentPreview].option def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) + def upsert(r: RAttachmentPreview): ConnectionIO[Option[FileKey]] = + OptionT(findById(r.id)) + .semiflatMap(existing => + update(existing.copy(fileId = r.fileId, name = r.name)).as(Some(existing.fileId)) + ) + .getOrElseF(insert(r).as(None)) + def findByIdAndCollective( attachId: Ident, collective: Ident diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 17faebba..2ac753ca 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -309,20 +309,22 @@ object RItem { def updateFolder( itemId: Ident, coll: Ident, - folderId: Option[Ident] - ): ConnectionIO[Int] = + folderIdOrName: Option[String] + ): ConnectionIO[(Int, Option[Ident])] = for { t <- currentTime - fid <- folderId match { - case Some(f) => RFolder.requireIdByIdOrName(f, f.id, coll).map(_.some) - case None => None.pure[ConnectionIO] + fid <- folderIdOrName match { + case Some(f) => + val fid = Ident.fromString(f).getOrElse(Ident.unsafe("")) + RFolder.requireIdByIdOrName(fid, f, coll).map(_.some) + case None => None.pure[ConnectionIO] } n <- DML.update( T, T.cid === coll && T.id === itemId, DML.set(T.folder.setTo(fid), T.updated.setTo(t)) ) - } yield n + } yield (n, fid) def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = for { @@ -334,6 +336,26 @@ object RItem { ) } yield n + def appendNotes( + itemId: Ident, + cid: Ident, + text: String, + sep: Option[String] + ): ConnectionIO[Option[String]] = { + val curNotes = + Select(select(T.notes), from(T), T.cid === cid && T.id === itemId).build + .query[Option[String]] + .option + + curNotes.flatMap { + case Some(notes) => + val newText = notes.map(_ + sep.getOrElse("")).getOrElse("") + text + updateNotes(itemId, cid, Some(newText)).as(newText.some) + case None => + (None: Option[String]).pure[ConnectionIO] + } + } + def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = for { t <- currentTime diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index fdcb8dd4..ce3c1daa 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -16,6 +16,7 @@ import docspell.store.qb._ import doobie._ import doobie.implicits._ +import scodec.bits.ByteVector case class RNode( id: Ident, @@ -23,13 +24,19 @@ case class RNode( url: LenientUri, updated: Timestamp, created: Timestamp, - notFound: Int + notFound: Int, + serverSecret: Option[ByteVector] ) {} object RNode { - def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] = - Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0)) + def apply[F[_]: Sync]( + id: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[RNode] = + Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0, serverSecret)) final case class Table(alias: Option[String]) extends TableDef { val tableName = "node" @@ -40,7 +47,9 @@ object RNode { val updated = Column[Timestamp]("updated", this) val created = Column[Timestamp]("created", this) val notFound = Column[Int]("not_found", this) - val all = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created, notFound) + val serverSecret = Column[ByteVector]("server_secret", this) + val all = NonEmptyList + .of[Column[_]](id, nodeType, url, updated, created, notFound, serverSecret) } def as(alias: String): Table = @@ -52,7 +61,7 @@ object RNode { DML.insert( t, t.all, - fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound}" + fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound},${v.serverSecret}" ) } @@ -65,6 +74,7 @@ object RNode { DML.set( t.nodeType.setTo(v.nodeType), t.url.setTo(v.url), + t.serverSecret.setTo(v.serverSecret), t.updated.setTo(v.updated) ) ) diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index c100f19d..651c99fe 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -152,6 +152,14 @@ object RUser { .query[Ident] .option + case class IdAndLogin(uid: Ident, login: Ident) + def getIdByIdOrLogin(idOrLogin: Ident): ConnectionIO[Option[IdAndLogin]] = + Select( + select(T.uid, T.login), + from(T), + T.uid === idOrLogin || T.login === idOrLogin + ).build.query[IdAndLogin].option + def getIdByAccount(account: AccountId): ConnectionIO[Ident] = OptionT(findIdByAccount(account)).getOrElseF( Sync[ConnectionIO].raiseError( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cd832f82..372e708e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -264,6 +264,9 @@ object Dependencies { val circeGenericExtra = Seq( "io.circe" %% "circe-generic-extras" % CirceVersion ) + val circeYaml = Seq( + "io.circe" %% "circe-yaml" % CirceVersion + ) // // https://github.com/Log4s/log4s;ASL 2.0 // val loggingApi = Seq(