From 7fdd78ad06c44180b4ffdb42d2acd5cac82aae36 Mon Sep 17 00:00:00 2001 From: eikek Date: Fri, 22 Apr 2022 14:07:28 +0200 Subject: [PATCH 1/2] Experiment with addons Addons allow to execute external programs in some context inside docspell. Currently it is possible to run them after processing files. Addons are provided by URLs to zip files. --- build.sbt | 44 +- .../scala/docspell/addons/AddonArchive.scala | 90 ++++ .../addons/AddonExecutionResult.scala | 33 ++ .../scala/docspell/addons/AddonExecutor.scala | 121 +++++ .../docspell/addons/AddonExecutorConfig.scala | 45 ++ .../addons/AddonLoggerExtension.scala | 28 ++ .../scala/docspell/addons/AddonMeta.scala | 216 +++++++++ .../main/scala/docspell/addons/AddonRef.scala | 9 + .../scala/docspell/addons/AddonResult.scala | 114 +++++ .../scala/docspell/addons/AddonRunner.scala | 110 +++++ .../docspell/addons/AddonTriggerType.scala | 55 +++ .../main/scala/docspell/addons/Context.scala | 72 +++ .../scala/docspell/addons/Directory.scala | 74 +++ .../main/scala/docspell/addons/InputEnv.scala | 32 ++ .../scala/docspell/addons/Middleware.scala | 43 ++ .../scala/docspell/addons/RunnerType.scala | 69 +++ .../docspell/addons/out/AddonOutput.scala | 44 ++ .../scala/docspell/addons/out/ItemFile.scala | 102 +++++ .../scala/docspell/addons/out/NewFile.scala | 77 ++++ .../scala/docspell/addons/out/NewItem.scala | 92 ++++ .../main/scala/docspell/addons/package.scala | 15 + .../docspell/addons/runner/CollectOut.scala | 43 ++ .../addons/runner/DockerBuilder.scala | 82 ++++ .../docspell/addons/runner/DockerRunner.scala | 93 ++++ .../addons/runner/NSpawnBuilder.scala | 83 ++++ .../addons/runner/NixFlakeRunner.scala | 123 +++++ .../docspell/addons/runner/RunnerUtil.scala | 171 +++++++ .../addons/runner/TrivialRunner.scala | 78 ++++ .../resources/docspell-dummy-addon-master.zip | Bin 0 -> 7634 bytes .../docspell/addons/AddonArchiveTest.scala | 44 ++ .../docspell/addons/AddonExecutorTest.scala | 63 +++ .../scala/docspell/addons/AddonMetaTest.scala | 37 ++ .../docspell/addons/AddonOutputTest.scala | 40 ++ .../docspell/addons/AddonRunnerTest.scala | 98 ++++ .../test/scala/docspell/addons/Fixtures.scala | 71 +++ .../classifier/StanfordTextClassifier.scala | 2 +- .../docspell/analysis/nlp/PipelineCache.scala | 1 + .../StanfordTextClassifierSuite.scala | 1 + .../nlp/StanfordNerAnnotatorSuite.scala | 1 + .../docspell/backend/AttachedEvent.scala | 1 + .../scala/docspell/backend/BackendApp.scala | 30 +- .../docspell/backend/BackendCommands.scala | 175 +++++++ .../main/scala/docspell/backend/Config.scala | 20 +- .../scala/docspell/backend/JobFactory.scala | 20 + .../backend/fulltext/CreateIndex.scala | 9 +- .../backend/joex/AddonEnvConfig.scala | 17 + .../docspell/backend/joex/AddonOps.scala | 199 ++++++++ .../backend/joex/AddonPostProcess.scala | 198 ++++++++ .../docspell/backend/joex/AddonPrepare.scala | 75 +++ .../backend/joex/LoggerExtension.scala | 18 + .../backend/ops/AddonRunConfigError.scala | 48 ++ .../backend/ops/AddonRunConfigValidate.scala | 54 +++ .../docspell/backend/ops/AddonValidate.scala | 156 +++++++ .../backend/ops/AddonValidationError.scala | 85 ++++ .../scala/docspell/backend/ops/OAddons.scala | 426 ++++++++++++++++++ .../docspell/backend/ops/OAttachment.scala | 223 +++++++++ .../scala/docspell/backend/ops/OItem.scala | 90 +++- .../scala/docspell/backend/ops/OJoex.scala | 28 +- .../scala/docspell/backend/ops/ONode.scala | 40 +- .../docspell/common/BaseJsonCodecs.scala | 10 +- .../main/scala/docspell/common/Binary.scala | 12 + .../scala/docspell/common/FileCategory.scala | 4 +- .../src/main/scala/docspell/common/Glob.scala | 21 +- .../main/scala/docspell/common/Ident.scala | 3 + .../docspell/common/ItemAddonTaskArgs.scala | 28 ++ .../scala/docspell/common/MimeTypeHint.scala | 5 + .../docspell/common/ProcessItemArgs.scala | 2 +- .../common/ScheduledAddonTaskArgs.scala | 19 + .../scala/docspell/common/SystemCommand.scala | 69 ++- .../scala/docspell/common/UrlMatcher.scala | 94 ++++ .../scala/docspell/common/UrlReader.scala | 35 ++ .../docspell/common/bc/AttachmentAction.scala | 30 ++ .../docspell/common/bc/BackendCommand.scala | 44 ++ .../common/bc/BackendCommandRunner.scala | 17 + .../scala/docspell/common/bc/ItemAction.scala | 102 +++++ .../scala/docspell/common/exec/Args.scala | 49 ++ .../main/scala/docspell/common/exec/Env.scala | 37 ++ .../scala/docspell/common/exec/SysCmd.scala | 43 ++ .../scala/docspell/common/exec/SysExec.scala | 163 +++++++ .../docspell/common/{ => util}/File.scala | 13 +- .../scala/docspell/common/util/Random.scala | 27 ++ .../test/scala/docspell/common/GlobTest.scala | 2 + .../docspell/common/UrlMatcherTest.scala | 60 +++ .../common/bc/BackendCommandTest.scala | 85 ++++ .../scala/docspell/config/Implicits.scala | 12 + .../docspell/convert/extern/ExternConv.scala | 1 + .../docspell/convert/ConversionTest.scala | 1 + .../scala/docspell/convert/FileChecks.scala | 1 + .../convert/extern/ExternConvTest.scala | 1 + .../main/scala/docspell/extract/ocr/Ocr.scala | 1 + .../docspell/extract/ocr/OcrConfig.scala | 1 + .../scala/docspell/files/FileSupport.scala | 61 +++ .../src/main/scala/docspell/files/Zip.scala | 67 ++- .../files/src/test/resources/zip-dirs-one.zip | Bin 0 -> 1306 bytes modules/files/src/test/resources/zip-dirs.zip | Bin 0 -> 1098 bytes .../test/scala/docspell/files/ZipTest.scala | 19 +- .../joex/src/main/resources/reference.conf | 71 +++ .../src/main/scala/docspell/joex/Config.scala | 4 +- .../scala/docspell/joex/JoexAppImpl.scala | 2 + .../main/scala/docspell/joex/JoexServer.scala | 2 +- .../main/scala/docspell/joex/JoexTasks.scala | 52 ++- .../joex/addon/AddonTaskExtension.scala | 30 ++ .../joex/addon/GenericItemAddonTask.scala | 130 ++++++ .../docspell/joex/addon/ItemAddonTask.scala | 76 ++++ .../scala/docspell/joex/addon/Result.scala | 40 ++ .../joex/addon/ScheduledAddonTask.scala | 39 ++ .../docspell/joex/analysis/NerFile.scala | 1 + .../docspell/joex/analysis/RegexNerFile.scala | 1 + .../scala/docspell/joex/learn/Classify.scala | 1 + .../multiupload/MultiUploadArchiveTask.scala | 2 +- .../joex/process/AttachmentPreview.scala | 2 +- .../joex/process/ExtractArchive.scala | 2 +- .../docspell/joex/process/ItemData.scala | 21 +- .../docspell/joex/process/ItemHandler.scala | 27 +- .../docspell/joex/process/ProcessItem.scala | 4 + .../docspell/joex/process/ReProcessItem.scala | 40 +- .../docspell/joex/process/RunAddons.scala | 43 ++ .../docspell/joex/process/SetGivenData.scala | 2 +- .../docspell/joex/routes/JoexRoutes.scala | 9 +- .../src/main/resources/joex-openapi.yml | 37 ++ .../docspell/joexapi/client/JoexClient.scala | 10 +- .../docspell/notification/api/Event.scala | 4 +- .../src/main/resources/docspell-openapi.yml | 379 ++++++++++++++++ .../src/main/resources/reference.conf | 34 ++ .../docspell/restserver/RestAppImpl.scala | 15 +- .../docspell/restserver/RestServer.scala | 10 + .../conv/AddonValidationSupport.scala | 70 +++ .../restserver/http4s/Responses.scala | 4 + .../http4s/ThrowableResponseMapper.scala | 42 ++ .../routes/AddonArchiveRoutes.scala | 127 ++++++ .../restserver/routes/AddonRoutes.scala | 39 ++ .../routes/AddonRunConfigRoutes.scala | 110 +++++ .../restserver/routes/AddonRunRoutes.scala | 40 ++ .../restserver/routes/ItemMultiRoutes.scala | 6 +- .../restserver/routes/ItemRoutes.scala | 2 +- .../docspell/restserver/webapp/Flags.scala | 6 +- .../docspell/restserver/ws/Background.scala | 43 ++ .../docspell/restserver/ws/OutputEvent.scala | 24 + .../restserver/ws/OutputEventEncoder.scala | 18 + .../scala/docspell/scheduler/Context.scala | 6 + .../docspell/scheduler/PermanentError.scala | 26 ++ .../main/scala/docspell/scheduler/Task.scala | 5 + .../scheduler/usertask/UserTaskScope.scala | 3 + .../docspell/scheduler/impl/ContextImpl.scala | 13 + .../docspell/scheduler/impl/LogSink.scala | 1 + .../docspell/scheduler/impl/QueueLogger.scala | 5 +- .../scheduler/impl/SchedulerImpl.scala | 33 +- .../db/migration/h2/V1.36.0__addons.sql | 47 ++ .../db/migration/mariadb/V1.36.0__addons.sql | 47 ++ .../migration/postgresql/V1.36.0__addons.sql | 47 ++ .../docspell/store/file/FileUrlReader.scala | 64 +++ .../docspell/store/impl/DoobieMeta.scala | 11 +- .../docspell/store/queries/AttachedFile.scala | 31 ++ .../docspell/store/queries/QAttachment.scala | 41 +- .../store/records/AddonRunConfigData.scala | 155 +++++++ .../records/AddonRunConfigResolved.scala | 72 +++ .../store/records/RAddonArchive.scala | 184 ++++++++ .../store/records/RAddonRunConfig.scala | 99 ++++ .../store/records/RAddonRunConfigAddon.scala | 68 +++ .../records/RAddonRunConfigTrigger.scala | 62 +++ .../store/records/RAttachmentMeta.scala | 3 + .../store/records/RAttachmentPreview.scala | 20 +- .../scala/docspell/store/records/RItem.scala | 34 +- .../scala/docspell/store/records/RNode.scala | 20 +- .../scala/docspell/store/records/RUser.scala | 8 + project/Dependencies.scala | 3 + 166 files changed, 8181 insertions(+), 115 deletions(-) create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Context.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Directory.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/Middleware.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/package.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala create mode 100644 modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala create mode 100644 modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala create mode 100644 modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/BackendCommands.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala create mode 100644 modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala create mode 100644 modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala create mode 100644 modules/common/src/main/scala/docspell/common/UrlMatcher.scala create mode 100644 modules/common/src/main/scala/docspell/common/UrlReader.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala create mode 100644 modules/common/src/main/scala/docspell/common/bc/ItemAction.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/Args.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/Env.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/SysCmd.scala create mode 100644 modules/common/src/main/scala/docspell/common/exec/SysExec.scala rename modules/common/src/main/scala/docspell/common/{ => util}/File.scala (91%) create mode 100644 modules/common/src/main/scala/docspell/common/util/Random.scala create mode 100644 modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala create mode 100644 modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala create mode 100644 modules/files/src/main/scala/docspell/files/FileSupport.scala create mode 100644 modules/files/src/test/resources/zip-dirs-one.zip create mode 100644 modules/files/src/test/resources/zip-dirs.zip create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/Result.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala create mode 100644 modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql create mode 100644 modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql create mode 100644 modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala create mode 100644 modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala create mode 100644 modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala diff --git a/build.sbt b/build.sbt index 2240a4e0..8156fc49 100644 --- a/build.sbt +++ b/build.sbt @@ -293,6 +293,15 @@ val openapiScalaSettings = Seq( field.copy(typeDef = TypeDef("DownloadState", Imports("docspell.common.DownloadState")) ) + case "addon-trigger-type" => + field => + field.copy(typeDef = + TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType")) + ) + case "addon-runner-type" => + field => + field + .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType"))) }) ) @@ -325,6 +334,7 @@ val common = project libraryDependencies ++= Dependencies.fs2 ++ Dependencies.circe ++ + Dependencies.circeGenericExtra ++ Dependencies.calevCore ++ Dependencies.calevCirce ) @@ -351,7 +361,7 @@ val files = project .in(file("modules/files")) .disablePlugins(RevolverPlugin) .settings(sharedSettings) - .withTestSettings + .withTestSettingsDependsOn(loggingScribe) .settings( name := "docspell-files", libraryDependencies ++= @@ -448,6 +458,19 @@ val notificationApi = project ) .dependsOn(common, loggingScribe) +val addonlib = project + .in(file("modules/addonlib")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .withTestSettingsDependsOn(loggingScribe) + .settings( + libraryDependencies ++= + Dependencies.fs2 ++ + Dependencies.circe ++ + Dependencies.circeYaml + ) + .dependsOn(common, files, loggingScribe) + val store = project .in(file("modules/store")) .disablePlugins(RevolverPlugin) @@ -469,7 +492,16 @@ val store = project libraryDependencies ++= Dependencies.testContainer.map(_ % Test) ) - .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe) + .dependsOn( + common, + addonlib, + query.jvm, + totp, + files, + notificationApi, + jsonminiq, + loggingScribe + ) val notificationImpl = project .in(file("modules/notification/impl")) @@ -647,7 +679,7 @@ val restapi = project openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, query.jvm, notificationApi, jsonminiq) + .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib) val joexapi = project .in(file("modules/joexapi")) @@ -667,7 +699,7 @@ val joexapi = project openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml", openapiStaticGen := OpenApiDocGenerator.Redoc ) - .dependsOn(common, loggingScribe) + .dependsOn(common, loggingScribe, addonlib) val backend = project .in(file("modules/backend")) @@ -683,6 +715,7 @@ val backend = project Dependencies.emil ) .dependsOn( + addonlib, store, notificationApi, joexapi, @@ -739,7 +772,7 @@ val config = project Dependencies.fs2 ++ Dependencies.pureconfig ) - .dependsOn(common, loggingApi, ftspsql, store) + .dependsOn(common, loggingApi, ftspsql, store, addonlib) // --- Application(s) @@ -946,6 +979,7 @@ val root = project ) .aggregate( common, + addonlib, loggingApi, loggingScribe, config, diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala new file mode 100644 index 00000000..3c2d6051 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.Zip + +final case class AddonArchive(url: LenientUri, name: String, version: String) { + def nameAndVersion: String = + s"$name-$version" + + def extractTo[F[_]: Async]( + reader: UrlReader[F], + directory: Path, + withSubdir: Boolean = true, + glob: Glob = Glob.all + ): F[Path] = { + val logger = docspell.logging.getLogger[F] + val target = + if (withSubdir) directory.absolute / nameAndVersion + else directory.absolute + + Files[F] + .exists(target) + .flatMap { + case true => target.pure[F] + case false => + Files[F].createDirectories(target) *> + reader(url) + .through(Zip.unzip(8192, glob)) + .through(Zip.saveTo(logger, target, moveUp = true)) + .compile + .drain + .as(target) + } + } + + /** Read meta either from the given directory or extract the url to find the metadata + * file to read + */ + def readMeta[F[_]: Async]( + urlReader: UrlReader[F], + directory: Option[Path] = None + ): F[AddonMeta] = + directory + .map(AddonMeta.findInDirectory[F]) + .getOrElse(AddonMeta.findInZip(urlReader(url))) +} + +object AddonArchive { + def read[F[_]: Async]( + url: LenientUri, + urlReader: UrlReader[F], + extractDir: Option[Path] = None + ): F[AddonArchive] = { + val addon = AddonArchive(url, "", "") + addon + .readMeta(urlReader, extractDir) + .map(m => addon.copy(name = m.meta.name, version = m.meta.version)) + } + + def dockerAndFlakeExists[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[(Boolean, Boolean)] = { + val files = Files[F] + def forPath(path: Path): F[(Boolean, Boolean)] = + (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled + + def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] = + data + .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix"))) + .collect { + case bin if bin.name == "Dockerfile" => (true, false) + case bin if bin.name == "flake.nix" => (false, true) + } + .compile + .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2)) + + archive.fold(forPath, forZip) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala new file mode 100644 index 00000000..08b3fd7b --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid +import cats.syntax.all._ + +case class AddonExecutionResult( + addonResults: List[AddonResult], + pure: Boolean +) { + def addonResult: AddonResult = addonResults.combineAll + def isFailure: Boolean = addonResult.isFailure + def isSuccess: Boolean = addonResult.isSuccess +} + +object AddonExecutionResult { + val empty: AddonExecutionResult = + AddonExecutionResult(Nil, false) + + def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult = + AddonExecutionResult( + a.addonResults ::: b.addonResults, + a.pure && b.pure + ) + + implicit val executionResultMonoid: Monoid[AddonExecutionResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala new file mode 100644 index 00000000..f13af4ef --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.Kleisli +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file._ + +import docspell.common.UrlReader +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonExecutor[F[_]] { + + def config: AddonExecutorConfig + + def execute(logger: Logger[F]): AddonExec[F] + + def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] = + execute(logger).run(in) +} + +object AddonExecutor { + + def apply[F[_]: Async]( + cfg: AddonExecutorConfig, + urlReader: UrlReader[F] + ): AddonExecutor[F] = + new AddonExecutor[F] with AddonLoggerExtension { + val config = cfg + + def execute(logger: Logger[F]): AddonExec[F] = + Kleisli { in => + for { + _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}") + ctx <- prepareDirectory( + logger, + in.baseDir, + in.outputDir, + in.cacheDir, + in.addons + ) + rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c)) + pure = ctx.foldl(true)((b, c) => b && c.meta.isPure) + } yield AddonExecutionResult(rs, pure) + } + + private def prepareDirectory( + logger: Logger[F], + baseDir: Path, + outDir: Path, + cacheDir: Path, + addons: List[AddonRef] + ): F[List[Context]] = + for { + addonsDir <- Directory.create(baseDir / "addons") + _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir) + _ <- Context + .userInputFile(baseDir) + .parent + .fold(().pure[F])(Files[F].createDirectories) + archives = addons.map(_.archive).distinctBy(_.url) + _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir") + mkCtxs <- archives.traverse { archive => + for { + _ <- logger.debug(s"Extracting $archive") + addonDir <- archive.extractTo(urlReader, addonsDir) + meta <- AddonMeta.findInDirectory(addonDir) + mkCtx = (ref: AddonRef) => + Context(ref, meta, baseDir, addonDir, outDir, cacheDir) + } yield archive.url -> mkCtx + } + ctxFactory = mkCtxs.toMap + res = addons.map(ref => ctxFactory(ref.archive.url)(ref)) + } yield res + + private def runAddon(logger: Logger[F], env: Env)( + ctx: Context + ): F[AddonResult] = + for { + _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}") + _ <- logger.trace("Storing user input into file") + _ <- Stream + .emit(ctx.addon.args) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(ctx.userInputFile, Flags.Write)) + .compile + .drain + + runner <- selectRunner(cfg, ctx.meta, ctx.addonDir) + result <- runner.run(logger, env, ctx) + } yield result + } + + def selectRunner[F[_]: Async]( + cfg: AddonExecutorConfig, + meta: AddonMeta, + addonDir: Path + ): F[AddonRunner[F]] = + for { + addonRunner <- meta.enabledTypes(Left(addonDir)) + // intersect on list retains order in first + possibleRunner = cfg.runner + .intersect(addonRunner) + .map(AddonRunner.forType[F](cfg)) + runner = possibleRunner match { + case Nil => + AddonRunner.failWith( + s"No runner available for addon config ${meta.runner} and config ${cfg.runner}." + ) + case list => + AddonRunner.firstSuccessful(list) + } + } yield runner +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala new file mode 100644 index 00000000..b4c29cb1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonExecutorConfig.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.AddonExecutorConfig._ +import docspell.common.Duration +import docspell.common.exec.{Args, SysCmd} + +case class AddonExecutorConfig( + runner: List[RunnerType], + runTimeout: Duration, + nspawn: NSpawn, + nixRunner: NixConfig, + dockerRunner: DockerConfig +) + +object AddonExecutorConfig { + + case class NSpawn( + enabled: Boolean, + sudoBinary: String, + nspawnBinary: String, + containerWait: Duration + ) { + val nspawnVersion = + SysCmd(nspawnBinary, Args.of("--version")).withTimeout(Duration.seconds(2)) + } + + case class NixConfig( + nixBinary: String, + buildTimeout: Duration + ) + + case class DockerConfig( + dockerBinary: String, + buildTimeout: Duration + ) { + def dockerBuild(imageName: String): SysCmd = + SysCmd(dockerBinary, "build", "-t", imageName, ".").withTimeout(buildTimeout) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala new file mode 100644 index 00000000..1b390985 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonLoggerExtension.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.logging.Logger + +trait AddonLoggerExtension { + + implicit final class LoggerAddonOps[F[_]](self: Logger[F]) { + private val addonName = "addon-name" + private val addonVersion = "addon-version" + + def withAddon(r: AddonArchive): Logger[F] = + self.capture(addonName, r.name).capture(addonVersion, r.version) + + def withAddon(r: Context): Logger[F] = + withAddon(r.addon.archive) + + def withAddon(m: AddonMeta): Logger[F] = + self.capture(addonName, m.meta.name).capture(addonVersion, m.meta.version) + } +} + +object AddonLoggerExtension extends AddonLoggerExtension diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala new file mode 100644 index 00000000..649b5d89 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.io.FileNotFoundException + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.Glob +import docspell.files.Zip + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.yaml.{parser => YamlParser} +import io.circe.{Decoder, Encoder} +import io.circe.{parser => JsonParser} + +case class AddonMeta( + meta: AddonMeta.Meta, + triggers: Option[Set[AddonTriggerType]], + args: Option[List[String]], + runner: Option[AddonMeta.Runner], + options: Option[AddonMeta.Options] +) { + + def nameAndVersion: String = + s"${meta.name}-${meta.version}" + + def parseResult: Boolean = + options.exists(_.collectOutput) + + def ignoreResult: Boolean = + !parseResult + + def isImpure: Boolean = + options.exists(_.isImpure) + + def isPure: Boolean = + options.forall(_.isPure) + + /** Returns a list of runner types that are possible to use for this addon. This is also + * inspecting the archive to return defaults when the addon isn't declaring it in the + * descriptor. + */ + def enabledTypes[F[_]: Async]( + archive: Either[Path, Stream[F, Byte]] + ): F[List[RunnerType]] = + for { + filesExists <- AddonArchive.dockerAndFlakeExists(archive) + (dockerFileExists, flakeFileExists) = filesExists + + nixEnabled = runner.flatMap(_.nix).map(_.enable) match { + case Some(flag) => flag + case None => flakeFileExists + } + + dockerEnabled = runner.flatMap(_.docker).map(_.enable) match { + case Some(flag) => flag + case None => dockerFileExists + } + + trivialEnabled = runner.flatMap(_.trivial).exists(_.enable) + + result = RunnerType.all.filter(_.fold(nixEnabled, dockerEnabled, trivialEnabled)) + } yield result + +} + +object AddonMeta { + + def empty(name: String, version: String): AddonMeta = + AddonMeta(Meta(name, version, None), None, None, None, None) + + case class Meta(name: String, version: String, description: Option[String]) + case class Runner( + nix: Option[NixRunner], + docker: Option[DockerRunner], + trivial: Option[TrivialRunner] + ) + case class NixRunner(enable: Boolean) + case class DockerRunner(enable: Boolean, image: Option[String], build: Option[String]) + case class TrivialRunner(enable: Boolean, exec: String) + case class Options(networking: Boolean, collectOutput: Boolean) { + def isPure = !networking && collectOutput + def isImpure = networking + def isUseless = !networking && !collectOutput + def isUseful = networking || collectOutput + } + + object NixRunner { + implicit val jsonEncoder: Encoder[NixRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[NixRunner] = + deriveDecoder + } + + object DockerRunner { + implicit val jsonEncoder: Encoder[DockerRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[DockerRunner] = + deriveDecoder + } + + object TrivialRunner { + implicit val jsonEncoder: Encoder[TrivialRunner] = + deriveEncoder + implicit val jsonDecoder: Decoder[TrivialRunner] = + deriveDecoder + } + + object Runner { + implicit val jsonEncoder: Encoder[Runner] = + deriveEncoder + implicit val jsonDecoder: Decoder[Runner] = + deriveDecoder + } + + object Options { + implicit val jsonEncoder: Encoder[Options] = + deriveEncoder + implicit val jsonDecoder: Decoder[Options] = + deriveDecoder + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = + deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = + deriveDecoder + } + + implicit val jsonEncoder: Encoder[AddonMeta] = + deriveEncoder + + implicit val jsonDecoder: Decoder[AddonMeta] = + deriveDecoder + + def fromJsonString(str: String): Either[Throwable, AddonMeta] = + JsonParser.decode[AddonMeta](str) + + def fromJsonBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromJsonString) + .rethrow + + def fromYamlString(str: String): Either[Throwable, AddonMeta] = + YamlParser.parse(str).flatMap(_.as[AddonMeta]) + + def fromYamlBytes[F[_]: Sync](bytes: Stream[F, Byte]): F[AddonMeta] = + bytes + .through(fs2.text.utf8.decode) + .compile + .string + .map(fromYamlString) + .rethrow + + def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = { + val logger = docspell.logging.getLogger[F] + val jsonFile = dir / "docspell-addon.json" + val yamlFile = dir / "docspell-addon.yaml" + val yamlFile2 = dir / "docspell-addon.yml" + + OptionT + .liftF(Files[F].exists(jsonFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading json addon file $jsonFile"))) + .flatMap(OptionT.whenF(_)(fromJsonBytes(Files[F].readAll(jsonFile)))) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile)))) + ) + .orElse( + OptionT + .liftF(Files[F].exists(yamlFile2)) + .flatTap(OptionT.whenF(_)(logger.debug(s"Reading yaml addon file $yamlFile2"))) + .flatMap(OptionT.whenF(_)(fromYamlBytes(Files[F].readAll(yamlFile2)))) + ) + .getOrElseF( + Sync[F].raiseError( + new FileNotFoundException(s"No docspell-addon.{yaml|json} file found in $dir!") + ) + ) + } + + def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = { + val fail: F[AddonMeta] = Async[F].raiseError( + new FileNotFoundException( + s"No docspell-addon.{yaml|json} file found in zip!" + ) + ) + zipFile + .through(Zip.unzip(8192, Glob("**/docspell-addon.*"))) + .filter(bin => !bin.name.endsWith("/")) + .flatMap { bin => + if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data)) + else if (bin.extensionIn(Set("yaml", "yml"))) + Stream.eval(AddonMeta.fromYamlBytes(bin.data)) + else Stream.empty + } + .take(1) + .compile + .last + .flatMap(_.map(Sync[F].pure).getOrElse(fail)) + } +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala new file mode 100644 index 00000000..37bc0106 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRef.scala @@ -0,0 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +case class AddonRef(archive: AddonArchive, args: String) diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala new file mode 100644 index 00000000..7efa9c26 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonResult.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monoid + +import docspell.addons.out.AddonOutput + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Codec, Decoder, Encoder} + +sealed trait AddonResult { + def toEither: Either[Throwable, AddonOutput] + + def isSuccess: Boolean = toEither.isRight + def isFailure: Boolean = !isSuccess + + def cast: AddonResult = this +} + +object AddonResult { + + /** The addon was run successful, but decoding its stdout failed. */ + case class DecodingError(message: String) extends AddonResult { + def toEither = Left(new IllegalStateException(message)) + } + object DecodingError { + implicit val jsonEncoder: Encoder[DecodingError] = deriveEncoder + implicit val jsonDecoder: Decoder[DecodingError] = deriveDecoder + } + + def decodingError(message: String): AddonResult = + DecodingError(message) + + def decodingError(ex: Throwable): AddonResult = + DecodingError(ex.getMessage) + + /** Running the addon resulted in an invalid return code (!= 0). */ + case class ExecutionError(rc: Int) extends AddonResult { + def toEither = Left(new IllegalStateException(s"Exit code: $rc")) + } + + object ExecutionError { + implicit val jsonEncoder: Encoder[ExecutionError] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionError] = deriveDecoder + } + + def executionError(rc: Int): AddonResult = + ExecutionError(rc) + + /** The execution of the addon failed with an exception. */ + case class ExecutionFailed(error: Throwable) extends AddonResult { + def toEither = Left(error) + } + + object ExecutionFailed { + implicit val throwableCodec: Codec[Throwable] = + Codec.from( + Decoder[String].emap(str => Right(ErrorMessageThrowable(str))), + Encoder[String].contramap(_.getMessage) + ) + + implicit val jsonEncoder: Encoder[ExecutionFailed] = deriveEncoder + implicit val jsonDecoder: Decoder[ExecutionFailed] = deriveDecoder + + private class ErrorMessageThrowable(msg: String) extends RuntimeException(msg) { + override def fillInStackTrace() = this + } + private object ErrorMessageThrowable { + def apply(str: String): Throwable = new ErrorMessageThrowable(str) + } + } + + def executionFailed(error: Throwable): AddonResult = + ExecutionFailed(error) + + /** The addon was run successfully and its output was decoded (if any). */ + case class Success(output: AddonOutput) extends AddonResult { + def toEither = Right(output) + } + + object Success { + implicit val jsonEncoder: Encoder[Success] = deriveEncoder + implicit val jsonDecoder: Decoder[Success] = deriveDecoder + } + + def success(output: AddonOutput): AddonResult = + Success(output) + + val empty: AddonResult = Success(AddonOutput.empty) + + def combine(a: AddonResult, b: AddonResult): AddonResult = + (a, b) match { + case (Success(o1), Success(o2)) => Success(AddonOutput.combine(o1, o2)) + case (Success(_), e) => e + case (e, Success(_)) => e + case _ => a + } + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("result").withKebabCaseConstructorNames + + implicit val jsonDecoder: Decoder[AddonResult] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonResult] = deriveConfiguredEncoder + + implicit val addonResultMonoid: Monoid[AddonResult] = + Monoid.instance(empty, combine) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala new file mode 100644 index 00000000..75efe4a2 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonRunner.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Applicative +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.runner._ +import docspell.common.exec.Env +import docspell.logging.Logger + +trait AddonRunner[F[_]] { + def runnerType: List[RunnerType] + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] +} + +object AddonRunner { + def forType[F[_]: Async](cfg: AddonExecutorConfig)(rt: RunnerType) = + rt match { + case RunnerType.NixFlake => NixFlakeRunner[F](cfg) + case RunnerType.Docker => DockerRunner[F](cfg) + case RunnerType.Trivial => TrivialRunner[F](cfg) + } + + def failWith[F[_]](errorMsg: String)(implicit F: Applicative[F]): AddonRunner[F] = + pure(AddonResult.executionFailed(new Exception(errorMsg))) + + def pure[F[_]: Applicative](result: AddonResult): AddonRunner[F] = + new AddonRunner[F] { + val runnerType = Nil + + def run(logger: Logger[F], env: Env, ctx: Context) = + Applicative[F].pure(result) + } + + def firstSuccessful[F[_]: Sync](runners: List[AddonRunner[F]]): AddonRunner[F] = + runners match { + case Nil => failWith("No runner available!") + case a :: Nil => a + case _ => + new AddonRunner[F] { + val runnerType = runners.flatMap(_.runnerType).distinct + + def run(logger: Logger[F], env: Env, ctx: Context) = + Stream + .emits(runners) + .evalTap(r => + logger.info( + s"Attempt to run addon ${ctx.meta.nameAndVersion} with runner ${r.runnerType}" + ) + ) + .evalMap(_.run(logger, env, ctx)) + .flatMap { + case r @ AddonResult.Success(_) => Stream.emit(r.cast.some) + case r @ AddonResult.ExecutionFailed(ex) => + if (ctx.meta.isPure) { + logger.stream + .warn(ex)(s"Addon runner failed, try next.") + .as(r.cast.some) + } else { + logger.stream.warn(ex)(s"Addon runner failed!").as(None) + } + case r @ AddonResult.ExecutionError(rc) => + if (ctx.meta.isPure) { + logger.stream + .warn(s"Addon runner returned non-zero: $rc. Try next.") + .as(r.cast.some) + } else { + logger.stream.warn(s"Addon runner returned non-zero: $rc!").as(None) + } + case AddonResult.DecodingError(message) => + // Don't retry as it is very unlikely that the output differs using another runner + // This is most likely a bug in the addon + logger.stream + .warn( + s"Error decoding the output of the addon ${ctx.meta.nameAndVersion}: $message. Stopping here. This is likely a bug in the addon." + ) + .as(None) + } + .unNoneTerminate + .takeThrough(_.isFailure) + .compile + .last + .flatMap { + case Some(r) => r.pure[F] + case None => + AddonResult + .executionFailed(new NoSuchElementException("No runner left :(")) + .pure[F] + } + } + } + + def firstSuccessful[F[_]: Sync]( + runner: AddonRunner[F], + runners: AddonRunner[F]* + ): AddonRunner[F] = + firstSuccessful(runner :: runners.toList) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala new file mode 100644 index 00000000..85438f09 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/AddonTriggerType.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList + +import io.circe.{Decoder, Encoder} + +sealed trait AddonTriggerType { + def name: String +} + +object AddonTriggerType { + + /** The final step when processing an item. */ + case object FinalProcessItem extends AddonTriggerType { + val name = "final-process-item" + } + + /** The final step when reprocessing an item. */ + case object FinalReprocessItem extends AddonTriggerType { + val name = "final-reprocess-item" + } + + /** Running periodically based on a schedule. */ + case object Scheduled extends AddonTriggerType { + val name = "scheduled" + } + + /** Running (manually) on some existing item. */ + case object ExistingItem extends AddonTriggerType { + val name = "existing-item" + } + + val all: NonEmptyList[AddonTriggerType] = + NonEmptyList.of(FinalProcessItem, FinalReprocessItem, Scheduled, ExistingItem) + + def fromString(str: String): Either[String, AddonTriggerType] = + all + .find(e => e.name.equalsIgnoreCase(str)) + .toRight(s"Invalid addon trigger type: $str") + + def unsafeFromString(str: String): AddonTriggerType = + fromString(str).fold(sys.error, identity) + + implicit val jsonEncoder: Encoder[AddonTriggerType] = + Encoder.encodeString.contramap(_.name) + + implicit val jsonDecoder: Decoder[AddonTriggerType] = + Decoder.decodeString.emap(fromString) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Context.scala b/modules/addonlib/src/main/scala/docspell/addons/Context.scala new file mode 100644 index 00000000..1c215e24 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Context.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import fs2.io.file.Path + +import docspell.common._ +import docspell.common.exec.{Args, Env, SysCmd} + +/** Context a list of addons is executed in. + * + * Each addon has its own `addonDir`, but all share the same `baseDir` in one run. + */ +case class Context( + addon: AddonRef, + meta: AddonMeta, + baseDir: Path, + addonDir: Path, + outputDir: Path, + cacheDir: Path +) { + def userInputFile = Context.userInputFile(baseDir) + def tempDir = Context.tempDir(baseDir) + + private[addons] def addonCommand( + binary: String, + timeout: Duration, + relativeToBase: Boolean, + outputDir: Option[String], + cacheDir: Option[String] + ): SysCmd = { + val execBin = Option + .when(relativeToBase)(binary) + .getOrElse((baseDir / binary).toString) + + val input = Option + .when(relativeToBase)(baseDir.relativize(userInputFile)) + .getOrElse(userInputFile) + + val allArgs = + Args(meta.args.getOrElse(Nil)).append(input) + val envAddonDir = Option + .when(relativeToBase)(baseDir.relativize(addonDir)) + .getOrElse(addonDir) + val envTmpDir = Option + .when(relativeToBase)(baseDir.relativize(tempDir)) + .getOrElse(tempDir) + val outDir = outputDir.getOrElse(this.outputDir.toString) + val cache = cacheDir.getOrElse(this.cacheDir.toString) + val moreEnv = + Env.of( + "ADDON_DIR" -> envAddonDir.toString, + "TMPDIR" -> envTmpDir.toString, + "TMP_DIR" -> envTmpDir.toString, + "OUTPUT_DIR" -> outDir, + "CACHE_DIR" -> cache + ) + + SysCmd(execBin, allArgs).withTimeout(timeout).addEnv(moreEnv) + } +} + +object Context { + def userInputFile(base: Path): Path = + base / "arguments" / "user-input" + def tempDir(base: Path): Path = + base / "temp" +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Directory.scala b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala new file mode 100644 index 00000000..3b72ac4c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Directory.scala @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import cats.{Applicative, Monad} +import fs2.io.file.{Files, Path, PosixPermissions} + +object Directory { + + def create[F[_]: Files: Applicative](dir: Path): F[Path] = + Files[F] + .createDirectories(dir, PosixPermissions.fromOctal("777")) + .as(dir) + + def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] = + (dir :: dirs.toList).traverse_(Files[F].createDirectories(_)) + + def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + List( + Files[F].isDirectory(dir), + Files[F].list(dir).take(1).compile.last.map(_.isDefined) + ).sequence.map(_.forall(identity)) + + def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] = + nonEmpty(dir).map(b => !b) + + def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + d <- mkTemp(parent, prefix) + } yield d + + def temp2[F[_]: Files]( + parent: Path, + prefix1: String, + prefix2: String + ): Resource[F, (Path, Path)] = + for { + _ <- Resource.eval(Files[F].createDirectories(parent)) + a <- mkTemp(parent, prefix1) + b <- mkTemp(parent, prefix2) + } yield (a, b) + + def createTemp[F[_]: Files: Monad]( + parent: Path, + prefix: String + ): F[Path] = + for { + _ <- Files[F].createDirectories(parent) + d <- mkTemp_(parent, prefix) + } yield d + + private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] = + Files[F] + .tempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) + + private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] = + Files[F] + .createTempDirectory( + parent.some, + prefix, + PosixPermissions.fromOctal("777") + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala new file mode 100644 index 00000000..bd5fb7cc --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect.Resource +import fs2.io.file.{Files, Path} + +import docspell.common.exec.Env + +case class InputEnv( + addons: List[AddonRef], + baseDir: Path, + outputDir: Path, + cacheDir: Path, + env: Env +) { + def addEnv(key: String, value: String): InputEnv = + copy(env = env.add(key, value)) + + def addEnv(vp: (String, String)*): InputEnv = + copy(env = env.addAll(vp.toMap)) + + def addEnv(vm: Map[String, String]): InputEnv = + copy(env = env ++ Env(vm)) + + def withTempBase[F[_]: Files]: Resource[F, InputEnv] = + Directory.temp(baseDir, "addon-").map(path => copy(baseDir = path)) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala new file mode 100644 index 00000000..4ca1a9fb --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/Middleware.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.Monad +import cats.data.Kleisli +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.Files + +trait Middleware[F[_]] extends (AddonExec[F] => AddonExec[F]) { self => + + def >>(next: Middleware[F]): Middleware[F] = + Middleware(self.andThen(next)) +} + +object Middleware { + def apply[F[_]](f: AddonExec[F] => AddonExec[F]): Middleware[F] = + a => f(a) + + def identity[F[_]]: Middleware[F] = Middleware(scala.Predef.identity) + + /** Uses a temporary base dir that is removed after execution. Use this as the last + * layer! + */ + def ephemeralRun[F[_]: Files: Sync]: Middleware[F] = + Middleware(a => Kleisli(_.withTempBase.use(a.run))) + + /** Prepare running an addon */ + def prepare[F[_]: Monad]( + prep: Kleisli[F, InputEnv, InputEnv] + ): Middleware[F] = + Middleware(a => Kleisli(in => prep.run(in).flatMap(a.run))) + + def postProcess[F[_]: Monad]( + post: Kleisli[F, AddonExecutionResult, Unit] + ): Middleware[F] = + Middleware(_.flatMapF(r => post.map(_ => r).run(r))) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala new file mode 100644 index 00000000..adf343e1 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/RunnerType.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.data.NonEmptyList +import cats.syntax.all._ + +import io.circe.{Decoder, Encoder} + +sealed trait RunnerType { + def name: String + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A +} +object RunnerType { + case object NixFlake extends RunnerType { + val name = "nix-flake" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = nixFlake + } + case object Docker extends RunnerType { + val name = "docker" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = docker + } + case object Trivial extends RunnerType { + val name = "trivial" + + def fold[A]( + nixFlake: => A, + docker: => A, + trivial: => A + ): A = trivial + } + + val all: NonEmptyList[RunnerType] = + NonEmptyList.of(NixFlake, Docker, Trivial) + + def fromString(str: String): Either[String, RunnerType] = + all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid runner value: $str") + + def unsafeFromString(str: String): RunnerType = + fromString(str).fold(sys.error, identity) + + def fromSeparatedString(str: String): Either[String, List[RunnerType]] = + str.split("[\\s,]+").toList.map(_.trim).traverse(fromString) + + implicit val jsonDecoder: Decoder[RunnerType] = + Decoder[String].emap(RunnerType.fromString) + + implicit val jsonEncoder: Encoder[RunnerType] = + Encoder[String].contramap(_.name) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala new file mode 100644 index 00000000..143e3dac --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/AddonOutput.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.kernel.Monoid + +import docspell.common.bc.BackendCommand + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Decoded stdout result from executing an addon. */ +case class AddonOutput( + commands: List[BackendCommand] = Nil, + files: List[ItemFile] = Nil, + newItems: List[NewItem] = Nil +) + +object AddonOutput { + val empty: AddonOutput = AddonOutput() + + def combine(a: AddonOutput, b: AddonOutput): AddonOutput = + AddonOutput(a.commands ++ b.commands, a.files ++ b.files) + + implicit val addonResultMonoid: Monoid[AddonOutput] = + Monoid.instance(empty, combine) + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonDecoder: Decoder[AddonOutput] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonOutput] = deriveConfiguredEncoder + + def fromString(str: String): Either[Throwable, AddonOutput] = + io.circe.parser.decode[AddonOutput](str) + + def unsafeFromString(str: String): AddonOutput = + fromString(str).fold(throw _, identity) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala new file mode 100644 index 00000000..bd470613 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/ItemFile.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.common._ +import docspell.files.FileSupport._ +import docspell.logging.Logger + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.{Decoder, Encoder} + +/** Addons can produce files in their output directory. These can be named here in order + * to do something with them. + * + * - textFiles will replace the extracted text with the contents of the file + * - pdfFiles will add/replace the converted pdf with the given file + * - previewImages will add/replace preview images + * - newFiles will be added as new attachments to the item + * + * Files must be referenced by attachment id. + */ +final case class ItemFile( + itemId: Ident, + textFiles: Map[String, String] = Map.empty, + pdfFiles: Map[String, String] = Map.empty, + previewImages: Map[String, String] = Map.empty, + newFiles: List[NewFile] = Nil +) { + def isEmpty: Boolean = + textFiles.isEmpty && pdfFiles.isEmpty && previewImages.isEmpty + + def nonEmpty: Boolean = !isEmpty + + def resolveTextFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.text("*"), textFiles) + + def resolvePdfFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.pdf, pdfFiles) + + def resolvePreviewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(String, Path)]] = + resolveFiles(logger, outputDir, MimeType.image("*"), previewImages) + + def resolveNewFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[List[(NewFile, Path)]] = + newFiles.traverseFilter(nf => + nf.resolveFile(logger, outputDir).map(_.map(p => (nf, p))) + ) + + private def resolveFiles[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path, + mime: MimeType, + files: Map[String, String] + ): F[List[(String, Path)]] = { + val allFiles = + files.toList.map(t => t._1 -> outputDir / t._2) + + allFiles.traverseFilter { case (key, file) => + OptionT(file.detectMime) + .flatMapF(fileType => + if (mime.matches(fileType)) (key -> file).some.pure[F] + else + logger + .warn( + s"File $file provided as ${mime.asString} file, but was recognized as ${fileType.asString}. Ignoring it." + ) + .as(None: Option[(String, Path)]) + ) + .value + } + } +} + +object ItemFile { + + implicit val jsonConfig: Configuration = + Configuration.default.withDefaults + + implicit val jsonEncoder: Encoder[ItemFile] = deriveConfiguredEncoder + implicit val jsonDecoder: Decoder[ItemFile] = deriveConfiguredDecoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala new file mode 100644 index 00000000..4807fe9c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewFile.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.effect.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewFile.Meta +import docspell.common.ProcessItemArgs.ProcessMeta +import docspell.common.{Ident, Language} +import docspell.logging.Logger + +import io.circe.Codec +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.deriveConfiguredCodec +import io.circe.generic.semiauto.deriveCodec + +case class NewFile(metadata: Meta = Meta.empty, file: String) { + + def resolveFile[F[_]: Files: Sync]( + logger: Logger[F], + outputDir: Path + ): F[Option[Path]] = { + val target = outputDir / file + Files[F] + .exists(target) + .flatMap(flag => + if (flag) target.some.pure[F] + else logger.warn(s"File not found: $file").as(Option.empty) + ) + } +} + +object NewFile { + + case class Meta( + language: Option[Language], + skipDuplicate: Option[Boolean], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessMeta( + cid: Ident, + itemId: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessMeta = + ProcessMeta( + collective = cid, + itemId = Some(itemId), + language = language.orElse(collLang).getOrElse(Language.English), + direction = None, + sourceAbbrev = sourceAbbrev, + folderId = None, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = None, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + val empty = Meta(None, None, None) + implicit val jsonCodec: Codec[Meta] = deriveCodec + } + + implicit val jsonConfig: Configuration = Configuration.default.withDefaults + + implicit val jsonCodec: Codec[NewFile] = deriveConfiguredCodec +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala new file mode 100644 index 00000000..c5511fd0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/out/NewItem.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.out + +import cats.Monad +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons.out.NewItem.Meta +import docspell.common._ +import docspell.logging.Logger + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +case class NewItem(metadata: Option[Meta], files: List[String]) { + + def toProcessMeta( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + metadata + .getOrElse(Meta(None, None, None, None, None, None, None)) + .toProcessArgs(cid, collLang, sourceAbbrev) + + def resolveFiles[F[_]: Files: Monad]( + logger: Logger[F], + outputDir: Path + ): F[List[Path]] = { + val allFiles = + files.map(name => outputDir / name) + + allFiles.traverseFilter { file => + Files[F] + .exists(file) + .flatMap { + case true => file.some.pure[F] + case false => + logger + .warn(s"File $file doesn't exist. Ignoring it.") + .as(None) + } + } + } +} + +object NewItem { + + case class Meta( + language: Option[Language], + direction: Option[Direction], + folderId: Option[Ident], + source: Option[String], + skipDuplicate: Option[Boolean], + tags: Option[List[String]], + attachmentsOnly: Option[Boolean] + ) { + + def toProcessArgs( + cid: Ident, + collLang: Option[Language], + sourceAbbrev: String + ): ProcessItemArgs.ProcessMeta = + ProcessItemArgs.ProcessMeta( + collective = cid, + itemId = None, + language = language.orElse(collLang).getOrElse(Language.English), + direction = direction, + sourceAbbrev = source.getOrElse(sourceAbbrev), + folderId = folderId, + validFileTypes = Seq.empty, + skipDuplicate = skipDuplicate.getOrElse(true), + fileFilter = None, + tags = tags, + reprocess = false, + attachmentsOnly = attachmentsOnly + ) + } + + object Meta { + implicit val jsonEncoder: Encoder[Meta] = deriveEncoder + implicit val jsonDecoder: Decoder[Meta] = deriveDecoder + } + + implicit val jsonDecoder: Decoder[NewItem] = deriveDecoder + implicit val jsonEncoder: Encoder[NewItem] = deriveEncoder +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/package.scala b/modules/addonlib/src/main/scala/docspell/addons/package.scala new file mode 100644 index 00000000..82a67232 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/package.scala @@ -0,0 +1,15 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell + +import cats.data.Kleisli + +package object addons { + + type AddonExec[F[_]] = Kleisli[F, InputEnv, AddonExecutionResult] + +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala new file mode 100644 index 00000000..48fa5b75 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/CollectOut.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.Applicative +import cats.effect.{Ref, Sync} +import cats.syntax.all._ +import fs2.Pipe + +trait CollectOut[F[_]] { + + def get: F[String] + + def append: Pipe[F, String, String] +} + +object CollectOut { + + def none[F[_]: Applicative]: CollectOut[F] = + new CollectOut[F] { + def get = "".pure[F] + def append = identity + } + + def buffer[F[_]: Sync]: F[CollectOut[F]] = + Ref + .of[F, Vector[String]](Vector.empty) + .map(buffer => + new CollectOut[F] { + override def get = + buffer.get.map(_.mkString("\n").trim) + + override def append = + _.evalTap(line => + if (line.trim.nonEmpty) buffer.update(_.appended(line)) else ().pure[F] + ) + } + ) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala new file mode 100644 index 00000000..b41d46e0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerBuilder.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} + +/** Builder for a docker system command. */ +case class DockerBuilder( + dockerBinary: String, + subCmd: String, + timeout: Duration, + containerName: Option[String] = None, + env: Env = Env.empty, + mounts: Args = Args.empty, + network: Option[String] = Some("host"), + workingDir: Option[String] = None, + imageName: Option[String] = None, + cntCmd: Args = Args.empty +) { + def containerCmd(args: Args): DockerBuilder = + copy(cntCmd = args) + def containerCmd(args: Seq[String]): DockerBuilder = + copy(cntCmd = Args(args)) + + def imageName(name: String): DockerBuilder = + copy(imageName = Some(name)) + + def workDirectory(dir: String): DockerBuilder = + copy(workingDir = Some(dir)) + + def withDockerBinary(bin: String): DockerBuilder = + copy(dockerBinary = bin) + + def withSubCmd(cmd: String): DockerBuilder = + copy(subCmd = cmd) + + def withEnv(key: String, value: String): DockerBuilder = + copy(env = env.add(key, value)) + + def withEnv(moreEnv: Env): DockerBuilder = + copy(env = env ++ moreEnv) + + def privateNetwork(flag: Boolean): DockerBuilder = + if (flag) copy(network = Some("none")) + else copy(network = Some("host")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): DockerBuilder = { + val target = cntDir.getOrElse(hostDir.toString) + val ro = Option.when(readOnly)(",readonly").getOrElse("") + val opt = s"type=bind,source=$hostDir,target=$target${ro}" + copy(mounts = mounts.append("--mount", opt)) + } + + def withName(containerName: String): DockerBuilder = + copy(containerName = Some(containerName)) + + def build: SysCmd = + SysCmd(dockerBinary, buildArgs).withTimeout(timeout) + + private def buildArgs: Args = + Args + .of(subCmd) + .append("--rm") + .option("--name", containerName) + .append(mounts) + .option("--network", network) + .append(env.mapConcat((k, v) => List("--env", s"${k}=${v}"))) + .option("-w", workingDir) + .appendOpt(imageName) + .append(cntCmd) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala new file mode 100644 index 00000000..a20ab5a0 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/DockerRunner.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonExecutorConfig.DockerConfig +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Env, SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +final class DockerRunner[F[_]: Async](cfg: DockerRunner.Config) extends AddonRunner[F] { + + val runnerType = List(RunnerType.Docker) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = for { + _ <- OptionT.whenF(requireBuild(ctx))(build(logger, ctx)).value + suffix <- Random[F].string(4) + cmd = createDockerCommand(env, ctx, suffix) + result <- RunnerUtil.runAddonCommand(logger, cmd, ctx) + } yield result + + def createDockerCommand( + env: Env, + ctx: Context, + suffix: String + ): SysCmd = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonArgs = + ctx.addonCommand( + "", + Duration.zero, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + + DockerBuilder(cfg.docker.dockerBinary, "run", cfg.timeout) + .withName(ctx.meta.nameAndVersion + "-" + suffix) + .withEnv(env) + .withEnv(addonArgs.env) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .workDirectory("/mnt/work") + .privateNetwork(ctx.meta.isPure) + .imageName(imageName(ctx)) + .containerCmd(addonArgs.args) + .build + } + + def build(logger: Logger[F], ctx: Context): F[Unit] = + for { + _ <- logger.info(s"Building docker image for addon ${ctx.meta.nameAndVersion}") + cmd = cfg.docker.dockerBuild(imageName(ctx)) + _ <- SysExec(cmd, logger, ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "docker build")) + .use(_.waitFor()) + _ <- logger.info(s"Docker image built successfully") + } yield () + + private def requireBuild(ctx: Context) = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .isEmpty + + private def imageName(ctx: Context): String = + ctx.meta.runner + .flatMap(_.docker) + .flatMap(_.image) + .getOrElse(s"${ctx.meta.meta.name}:latest") +} + +object DockerRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): DockerRunner[F] = + new DockerRunner[F](Config(cfg.dockerRunner, cfg.runTimeout)) + + case class Config(docker: DockerConfig, timeout: Duration) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala new file mode 100644 index 00000000..accd2059 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NSpawnBuilder.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import fs2.io.file.Path + +import docspell.common.exec.{Args, Env, SysCmd} + +case class NSpawnBuilder( + child: SysCmd, + chroot: Path, + spawnBinary: String = "systemd-nspawn", + sudoBinary: String = "sudo", + args: Args = Args.empty, + env: Env = Env.empty +) { + + def withNSpawnBinary(bin: String): NSpawnBuilder = + copy(spawnBinary = bin) + + def withSudoBinary(bin: String): NSpawnBuilder = + copy(sudoBinary = bin) + + def withEnv(key: String, value: String): NSpawnBuilder = + copy(args = args.append(s"--setenv=$key=$value")) + + def withEnvOpt(key: String, value: Option[String]): NSpawnBuilder = + value.map(v => withEnv(key, v)).getOrElse(this) + + def withName(containerName: String): NSpawnBuilder = + copy(args = args.append(s"--machine=$containerName")) + + def mount( + hostDir: Path, + cntDir: Option[String] = None, + readOnly: Boolean = true + ): NSpawnBuilder = { + val bind = if (readOnly) "--bind-ro" else "--bind" + val target = cntDir.map(dir => s":$dir").getOrElse("") + copy(args = args.append(s"${bind}=${hostDir}${target}")) + } + + def workDirectory(dir: String): NSpawnBuilder = + copy(args = args.append(s"--chdir=$dir")) + + def portMap(port: Int): NSpawnBuilder = + copy(args = args.append("-p", port.toString)) + + def privateNetwork(flag: Boolean): NSpawnBuilder = + if (flag) copy(args = args.append("--private-network")) + else this + + def build: SysCmd = + SysCmd( + program = if (sudoBinary.nonEmpty) sudoBinary else spawnBinary, + args = buildArgs, + timeout = child.timeout, + env = env + ) + + private def buildArgs: Args = + Args + .of("--private-users=identity") // can't use -U because need writeable bind mounts + .append("--notify-ready=yes") + .append("--ephemeral") + .append("--as-pid2") + .append("--console=pipe") + .append("--no-pager") + .append("--bind-ro=/bin") + .append("--bind-ro=/usr/bin") + .append("--bind-ro=/nix/store") + .append(s"--directory=$chroot") + .append(args) + .append(child.env.map((n, v) => s"--setenv=$n=$v")) + .prependWhen(sudoBinary.nonEmpty)(spawnBinary) + .append("--") + .append(child.program) + .append(child.args) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala new file mode 100644 index 00000000..ce12c73c --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/NixFlakeRunner.scala @@ -0,0 +1,123 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.addons.AddonExecutorConfig.{NSpawn, NixConfig} +import docspell.addons._ +import docspell.addons.runner.NixFlakeRunner.PreCtx +import docspell.common.Duration +import docspell.common.exec._ +import docspell.logging.Logger + +final class NixFlakeRunner[F[_]: Async](cfg: NixFlakeRunner.Config) + extends AddonRunner[F] { + + val runnerType = List(RunnerType.NixFlake) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ): F[AddonResult] = + prepare(logger, ctx) + .flatMap { preCtx => + if (preCtx.nspawnEnabled) runInContainer(logger, env, preCtx, ctx) + else runOnHost(logger, env, preCtx, ctx) + } + + def prepare(logger: Logger[F], ctx: Context): F[PreCtx] = + for { + _ <- logger.info(s"Prepare addon ${ctx.meta.nameAndVersion} for executing via nix") + _ <- logger.debug(s"Building with nix build") + _ <- SysExec(cfg.nixBuild, logger, workdir = ctx.addonDir.some) + .flatMap(_.logOutputs(logger, "nix build")) + .use(_.waitFor()) + bin <- findFile(ctx.addonDir / "result" / "bin", ctx.addonDir / "result") + _ <- logger.debug(s"Build done, found binary: $bin") + _ <- logger.debug(s"Checking for systemd-nspawn…") + cnt <- checkContainer(logger) + _ <- + if (cnt) + logger.debug(s"Using systemd-nspawn to run addon in a container.") + else + logger.info(s"Running via systemd-nspawn is disabled in the config file") + } yield PreCtx(cnt, ctx.baseDir.relativize(bin)) + + private def checkContainer(logger: Logger[F]): F[Boolean] = + if (!cfg.nspawn.enabled) false.pure[F] + else RunnerUtil.checkContainer(logger, cfg.nspawn) + + private def runOnHost( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = + SysCmd(preCtx.binary.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + + private def runInContainer( + logger: Logger[F], + env: Env, + preCtx: PreCtx, + ctx: Context + ): F[AddonResult] = { + val cmd = SysCmd(preCtx.binary.toString, Args.empty) + .withTimeout(cfg.timeout) + .addEnv(env) + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + } + + /** Find first file, try directories in given order. */ + private def findFile(firstDir: Path, more: Path*): F[Path] = { + val fail: F[Path] = Sync[F].raiseError( + new NoSuchElementException( + s"No file found to execute in ${firstDir :: more.toList}" + ) + ) + + Stream + .emits(more) + .cons1(firstDir) + .flatMap(dir => + Files[F] + .list(dir) + .evalFilter(p => Files[F].isDirectory(p).map(!_)) + .take(1) + ) + .take(1) + .compile + .last + .flatMap(_.fold(fail)(Sync[F].pure)) + } +} + +object NixFlakeRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): NixFlakeRunner[F] = + new NixFlakeRunner[F](Config(cfg.nixRunner, cfg.nspawn, cfg.runTimeout)) + + case class Config( + nix: NixConfig, + nspawn: NSpawn, + timeout: Duration + ) { + + val nixBuild = + SysCmd(nix.nixBinary, Args.of("build")).withTimeout(nix.buildTimeout) + + val nspawnVersion = nspawn.nspawnVersion + } + + case class PreCtx(nspawnEnabled: Boolean, binary: Path) +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala new file mode 100644 index 00000000..e404b442 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala @@ -0,0 +1,171 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect.{Async, Sync} +import cats.syntax.all._ +import fs2.Pipe +import fs2.io.file.Files + +import docspell.addons._ +import docspell.addons.out.AddonOutput +import docspell.common.exec.{SysCmd, SysExec} +import docspell.common.util.Random +import docspell.logging.Logger + +import io.circe.{parser => JsonParser} + +private[addons] object RunnerUtil { + + /** Run the given `cmd` on this machine. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runDirectly[F[_]: Async]( + logger: Logger[F], + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val addonCmd = ctx + .addonCommand(cmd.program, cmd.timeout, relativeToBase = false, None, None) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + runAddonCommand(logger, addonCmd, ctx) + } + + /** Run the given `cmd` inside a container via systemd-nspawn. + * + * The `cmd` is containing a template command to execute the addon. The path are + * expected to be relative to the `ctx.baseDir`. Additional arguments and environment + * variables are added as configured in the addon. + */ + def runInContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn, + ctx: Context + )(cmd: SysCmd): F[AddonResult] = { + val outputPath = "/mnt/output" + val cachePath = "/mnt/cache" + val addonCmd = ctx + .addonCommand( + cmd.program, + cmd.timeout, + relativeToBase = true, + outputPath.some, + cachePath.some + ) + .withArgs(_.append(cmd.args)) + .addEnv(cmd.env) + + val chroot = ctx.baseDir / "cnt-root" + val nspawn = NSpawnBuilder(addonCmd, chroot) + .withNSpawnBinary(cfg.nspawnBinary) + .withSudoBinary(cfg.sudoBinary) + .mount(ctx.baseDir, "/mnt/work".some, readOnly = false) + .mount(ctx.cacheDir, cachePath.some, readOnly = false) + .mount(ctx.outputDir, outputPath.some, readOnly = false) + .workDirectory("/mnt/work") + .withEnv("XDG_RUNTIME_DIR", "/mnt/work") + .privateNetwork(ctx.meta.isPure) + + for { + suffix <- Random[F].string(4) + _ <- List(chroot).traverse_(Files[F].createDirectories) + res <- runAddonCommand( + logger, + nspawn.withName(ctx.meta.nameAndVersion + "-" + suffix).build, + ctx + ) + // allow some time to unregister the current container + // only important when same addons are called in sequence too fast + _ <- Sync[F].sleep(cfg.containerWait.toScala) + } yield res + } + + private def procPipe[F[_]]( + p: String, + ctx: Context, + collect: CollectOut[F], + logger: Logger[F] + ): Pipe[F, String, Unit] = + _.through(collect.append) + .map(line => s">> [${ctx.meta.nameAndVersion} ($p)] $line") + .evalMap(logger.debug(_)) + + /** Runs the external command that is executing the addon. + * + * If the addons specifies to collect its output, the stdout is parsed as json and + * decoded into [[AddonOutput]]. + */ + def runAddonCommand[F[_]: Async]( + logger: Logger[F], + cmd: SysCmd, + ctx: Context + ): F[AddonResult] = + for { + stdout <- + if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F] + else CollectOut.none[F].pure[F] + cmdResult <- SysExec(cmd, logger, ctx.baseDir.some) + .flatMap( + _.consumeOutputs( + procPipe("out", ctx, stdout, logger), + procPipe("err", ctx, CollectOut.none[F], logger) + ) + ) + .use(_.waitFor()) + .attempt + addonResult <- cmdResult match { + case Right(rc) if rc != 0 => + for { + _ <- logger.error( + s"Addon ${ctx.meta.nameAndVersion} returned non-zero: $rc" + ) + } yield AddonResult.executionError(rc) + + case Right(_) => + for { + _ <- logger.debug(s"Addon ${ctx.meta.nameAndVersion} executed successfully!") + out <- stdout.get + _ <- logger.debug(s"Addon stdout: $out") + result = Option + .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)( + JsonParser + .decode[AddonOutput](out) + .fold(AddonResult.decodingError, AddonResult.success) + ) + .getOrElse(AddonResult.empty) + } yield result + + case Left(ex) => + logger + .error(ex)(s"Executing external command failed!") + .as(AddonResult.executionFailed(ex)) + } + } yield addonResult + + /** Check whether `systemd-nspawn` is available on this machine. */ + def checkContainer[F[_]: Async]( + logger: Logger[F], + cfg: AddonExecutorConfig.NSpawn + ): F[Boolean] = + for { + rc <- SysExec(cfg.nspawnVersion, logger) + .flatMap(_.logOutputs(logger, "nspawn")) + .use(_.waitFor()) + _ <- + OptionT + .whenF(rc != 0)( + logger.warn( + s"No systemd-nspawn found! Addon is not executed inside a container." + ) + ) + .value + } yield rc == 0 +} diff --git a/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala new file mode 100644 index 00000000..31ac2694 --- /dev/null +++ b/modules/addonlib/src/main/scala/docspell/addons/runner/TrivialRunner.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons.runner + +import cats.data.OptionT +import cats.effect._ +import cats.kernel.Monoid +import cats.syntax.all._ +import fs2.io.file.PosixPermission._ +import fs2.io.file.{Files, PosixPermissions} + +import docspell.addons.AddonExecutorConfig.NSpawn +import docspell.addons._ +import docspell.common.Duration +import docspell.common.exec.{Args, Env, SysCmd} +import docspell.logging.Logger + +final class TrivialRunner[F[_]: Async](cfg: TrivialRunner.Config) extends AddonRunner[F] { + private val sync = Async[F] + private val files = Files[F] + implicit val andMonoid: Monoid[Boolean] = Monoid.instance[Boolean](true, _ && _) + + private val executeBits = PosixPermissions( + OwnerExecute, + OwnerRead, + OwnerWrite, + GroupExecute, + GroupRead, + OthersExecute, + OthersRead + ) + + val runnerType = List(RunnerType.Trivial) + + def run( + logger: Logger[F], + env: Env, + ctx: Context + ) = { + val binaryPath = ctx.meta.runner + .flatMap(_.trivial) + .map(_.exec) + .map(bin => ctx.addonDir / bin) + + binaryPath match { + case None => + sync.raiseError(new IllegalStateException("No executable specified in addon!")) + + case Some(file) => + val bin = ctx.baseDir.relativize(file) + val cmd = SysCmd(bin.toString, Args.empty).withTimeout(cfg.timeout).addEnv(env) + + val withNSpawn = + OptionT + .whenF(cfg.nspawn.enabled)(RunnerUtil.checkContainer(logger, cfg.nspawn)) + .getOrElse(false) + + files.setPosixPermissions(file, executeBits).attempt *> + withNSpawn.flatMap { + case true => + RunnerUtil.runInContainer(logger, cfg.nspawn, ctx)(cmd) + case false => + RunnerUtil.runDirectly(logger, ctx)(cmd) + } + } + } +} + +object TrivialRunner { + def apply[F[_]: Async](cfg: AddonExecutorConfig): TrivialRunner[F] = + new TrivialRunner[F](Config(cfg.nspawn, cfg.runTimeout)) + + case class Config(nspawn: NSpawn, timeout: Duration) +} diff --git a/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip b/modules/addonlib/src/test/resources/docspell-dummy-addon-master.zip new file mode 100644 index 0000000000000000000000000000000000000000..20482861376733b293b75844b34dced90b3b330d GIT binary patch literal 7634 zcmb7p1yodB)b_vtBHg7DBPcl_-Q6&BO2Z62bVx`jLxTt-0us_9Eg&TzASn&f4NA8N z67u2e{~EmN|9xlHxif3lI?vtro@ej#?7h{LfM{5NuOr4XRqLN0|L;NyxD0?gTDUsF z?Csg0?r^vln;8`9=)kV6g$_X3A%UB#{jwHM1x<|VeC^js^QX07D;qZ(D+fmx*mo2c zn5(W*1L$(F$t?)rRBE6wx*Q4KVp9qFUN67hNlQLK5OcijGBokmG`%f2OS^>VvAVYwxA zXt;Ip51xFa4UyY;wJFkM)P*`Vhv&wdX~AvCso`32+YcB~H#isja1@cwu$7ae4<~y{ zTnXevLV2zyr_@Q0dpw}up|cYLid~6WG8siT-w>G;3t$@K>t-sDJ0QAQ0u?+Y0X;~d zlL3tgZQa3_oX0$TL&4fbh0s_r%^bt>N_8C8W06vtX#kr+RowE1X78}xwVKZgQn_5G z8=KGRP2wt_Q9GOYlDWzZ+EBdgC^Ck>HIWzNcozJ?H4*JB{h6fxXX|p^I_!$R`~v52 zGErm0SB=jt4QYIzHQU%(EYB)}!}Aqx%~rq4l9B0vKCU_&nm1CMG|3tygl5%|U9iQN@A6E_!i+IH_sxcKEztXg|s(_AQ)EXOE`R2niqk;dJEz zM9qh{=5;l7`o`?T+d!4+7!TBjB(Yx!AXsfyB5_VGne$2gCxU6nNJv9uz;Nhyerf7B zDJyjJYGNv|f#>G2I@(!v02$H<f7(5u+4S8g#-KNn{S zTa{=@kbE&UIp9|-Owvq>uA>70&Da3I%|BW3-EaQ!7_b-I{<}E{9f;!sFG1b0t}@?a zqu1&I)2kMxArAOYP@gC{T0VQ&A`8;?dvmLgW`<8{TxSwYhB+HbCED}KYwwwmH&vo# zPcYHM`n{Wv8Fyz|WHyD4Y%Omr_;|lhSllqwxIx1?He2TG=PK9M4^xRKeMR_mD!AQZ zB3kq#ELHU&XY2l1z`Ni;axu?_cFEoj*-EI<$t7&*8C1NXyIIGKhbz)jp+Q_P_Cg%) zUl-e(6{dFL_36CsQq&*oyL0V%lm8)0m``UQUdw)@c>E(wv?Z=K6oCaXq^QEqYp#0K z^j+~{OJ>RFoq)6)2iK4&{~+Y#-kW{+xY*Pl{dH5d9J#CN!x zs_3?+(K_x!_-4Jtls_PZShun#q!{@-I}~t`B#P4aEdmS~LPSR&zwPNg;;0HMHn0ov z$Mv$$rJZVzIeIm#i*UONi9bvtvVGghE{*Q)`J@Jdi55-5exeA#6qM$Y@_PYtw_eRt z*w3JndK^n4O=M46m%p?elZB#wpGotE@=S!lkUE^;fK3MR5GW<{nPvM*cWZQy>rHl^ zS(``Kck`lr+-VhGt}YlGxMNFXDul-jZ8Yy740+XFl>97-i__x z#_AVFmmat-Ip5D<+_vhZiJxH~%u;?&r-F7QWn8}S+V}$mU%1gRq*r#pn(Bp`AG;1{ z{DsrYfO)GqLNAaB8wJVGF6jq^>KjM!dugSBFl&F{y&~^%TOHCR$@g5%&Go&NL)rFx zQS)9iHhn@x@9xU&b%r|+?LCPm+dG*ZEv+AyMo;|ehIBmP4I{dN!6-S)j+Ud zz@+WG-G=VP-k;cI?zEG|sEC2?T>GUd2x*Y^C70!eQ*NKoWkqb&L8-KL2Tt?-Bjf6c z!m7eL-*7xAWQo@@v19QAEOxqtO@Ce22)`L$Un9DoIGZGtKQzvd%99}xsUTay2^KnyxkL;6( z&luaBxHa91$lLJ+Gb8m4^;S8G|{(ihrBTP^lebx5fy z>vl;KFy)J}m&O9mS`7?Mo^UEUwYYoP@*Do{+hqzF1pO(CZrc*|9>s=BN2HMv{yJtn4fZA+<^k59 zbdIXZdmB}F^J;U-@}FnU?KY3Rf9m-LgaJDVqb2c=F|)}tLXvohysBbaE~{-K5ka7;eO*)Anj7{wOeb_sdknrfbha z&*r9avfk2x-`HJNjFvucE*g?%4waBL!l$Jkn}_XsPmlfgAlXMTlCQ!9%z zORea8lb_m}lI@rGPN9o0D2QXjUA9=*f$8^$<}2?s21+24C34FY(E=>o1A(ez5SbxZQQjG zr_DJ;Isqo+q&F5uVT11W=Dw>L*8Pdq++Mq+xph!#NZ}g-jM5-Q7f6@{$1KLcYc&}Z za~#T$XKqe5wqIguRwf-14uX$10_pWfaV<#MXgY3ybfw4jLk|{66HRh&`j}yAElwqs z06qk9?tlv>!Y|>fchk=r$wSFxGM!c9KGKtntVRdlopp5<69-=^DK0btswBt5*fb`jH`;c(r$-lVCv#kiQIni`A2hl& z0hFStmkD`6cFixFUfI;eq3zN`3MkrRGb7hPPojz{8d zFEOTSQ5H@6YgT;HdoOF!?_GGifpKY%Avw9C(?dA!G(J)-kS?02i-H%6F$W7LHXSsh zFHYEc)wS)`I4F_mQRBV#Q?qr~>`V#1Ln413s3at zQTZ4fhq59-NtzgSpsu6CR6?q!V=c?`cNnv%OW~uzSh8ME(aK{4W*KqNoYYVj@6GG$ zgU$r1`iitT7vc}^qWK4xu^*lZIt}P1Epo?F)S--}gI@ z4bt$-P50b&vmRE~@WPP+^aK)LHs8H18q~_VnfqeqR;Hk1kO^Mvqh6EV3_Mg3XOI)N zi7`c@$6|N<0)s-XdXEESRud9Ti`f z8)&xV@x0wkvAC5RT(%Y`o3rZ|4KAib=x#i7_Vel+Iw^>{J*11uVf#q|b$@3i-mVBl zr7>UrQWo=iC`mX~QG~IAJgo4c3Yq(3$1<)XPutD)(=WYCVy#w73?Es;#Lh(Z!#$Uq z>A{h{OUFGXzj}Y`J<9gobJhO_?JuJ5oA*1|-1+YOQJNj+qW@fFv0y0T>OOyE-;n_Un>EIqfNib(rfw}0Wcz6FMS>gNO;jBrn$ryd{H+phe z`7BQJj45x^i5Lf-#(#Psh(;<02TApbJfGi2`4 zX7|7;B6X^}-X+1%Ydp?9j57tY17r$GJGoEBh`j8w?;R(=n{hvs zqXk^g5ko<1VR>qZ1=ILapSzQ}@sQD$_Jf`!^&B(waDBm~%+ASIWdpVRku)0ywNp0# z2Dpi@GMYKf`Q0o~Vg~#$)4;gJ4S393HZ7X2uN5J%vjO2sLGUDHZz#A2g0yv>M{<@A z&JnwTr4qCelxqyAyP3>Vpa$a1sNF`OKFyL@o#+`UEVxDEgQ95L+b0!B6Iu_qRqoDP z1Et1^y|1X$@Q2b~%AH{y*KglQl4A0+*m^XJS5bzf9JdXMQg7fWlMWG)P0V^2c3g6l z?mvCMqd|85Kuq_Tu#KMNvJsVQ4g88>1I!(zfbU6e+su=YG?jAlgnX~hJ6EsC*i^B~ z){s~3c?hiBI79iDHvdgzDgUI>&k6J2WcFLKDYX604E3CL_2(1xxAory{?F8Nk7=}h zLrW06{gSSrO8KzRiG0V~YN`%`?TwLu>;lygq! z_V%;4HVYb^C(OF3RHw@o%0zAi@eJVyD1olLA}P^S^V$8>pYAOCxy?CwMA-hi$^QNJ zoj%XdvH(TI!q$o+zY_jn?p~?XO4Bnf{+yac&XdOr%}{jVjRSvcqsn);ProNj|A(z` zb+P!pSqxK#$*>y8)t9sV_aqmj2F z@XUdF+Er?E6`Dwm9!qvmP)g!=VGxz^~`8` zhES=sQl$_uum&2ZzZw{LtKvrSwOA!4L(4`=CrVVX^r~s}BW>A(IX)Z6hIkx){{yw5 z;YOq{w={Y4qFh>vA>QGsNl$ceM?h|qGjEM3m6MHTPr?9KSp}{t&a%Ks$)yBKj5{Br z%E()N{fi48^G2Rw-CFljzi;yH_OT8EDZ58@0%M9T1>05P_dA5()ddC2S-bgp2vM21 zOW%`W(w5P3@g{W2u`EI1BRIRiY?FL7ZTkFn+n*MXPSzLZ(QlaC9rRwgV-bRWCrNpF zO`v&l-g_mG7th);Z5o;ubY`&#e#<|8HRYSS4ql=^qewlzEX!qPd*OT%}t$Uyy?3YUd^8liMaJYOqd`n`glU`Z?T*;=_uCt-J4Dm zEK+I^frLF)UH1MweFpq3q`u6~M*a0sIroWg;PX z_c$n|4@6q5=+Ma2*k5NT)V@aJXio*e%ah17!d-mrP9`89>Jm-mhFfAKC_&xVg8iy# zxv^k-9MtGRr4MpOh@7h0E=v=3;XkC!a-zunWHpX8f>OU{Y)yd5BmnSUL>R0*4k9F_36&J*aiEsTNTherYoRfvxWxL25K^j+UA5jjfV47 zzv^>`6DCdNXoa)!yc`za>;6>DnEJ9BjLJKwZtfRB8#B?t2!1Y31ws~$kbz=xn(t7o zg9)*^b9F}sKc~kEJqyp7w|y4`v52R{AG0uA?O1RD#c9e0q1C?3;HIaN6}pC<-Q4%N zd+MVaQx6q*7t#4RPMqiAAhu*WUj+TMaJ=>~!KOzYBgB|lGPG8-aWtKYqvBD9(IA^F zE5CR#!{tnRJlRZs>ZsmWj>xuLXGucNwSv~1TMj0&S?c!Jci4lbr2I@@+%+QBCvJ&< zD)Jgf9Tag=nNNScMr9)Q$ka+bZ9C;455Hm;p)=qe6~5CHB@FL%wvke;(gS#o@X#Q) zh}<~vSayT0P4(k=O@+-~*HGj<=9@_=3}M*Gh+4cNZ?p4;mEiGxNTKdE(X5n2>jMF6 z3twL9O9}FmaLTJ9k$nJ~O4k7ArDean1?atj%ui9fJEfeqszUCtL5w7y2i>b27iFXN zA&(KQ@0CvziVU+ln^IX22G z)=u=Rd~wG({Iqt$De`%;Ud6ch)8hX6QuF8T{g!nDZ&U;H+gh}2EDCsCMHNbV;YJ8B zw{{COcrUi__0^H;h6Zo`wCQddu%)iUs;K}^PS(|O9wgTQSQ=b)1wrXjcjd~_cmMrH zPbl9kp~m@Pf8(5@e>`RWq374j&;FPFP{Of^mLhoTBpkfwLReDzqdmH(PDxGY(+8)UTROr}>U;(t=(hM*qFJ$xU=gTh>&CK{{Ca za+D~4bT57E(=dAP5+h$Q`GkRn-uji9(k~-c(-YopoK(rm{lkdPH0!?k%~|P^HxAM0(cK7XeRBvY%YL+j|!YisR9;R4jFdsh$t8F>cW+*45cXjl2IxNv)SAq-X zV+rha#3qDVjWZV7HApQ&WM%py*X`|G2BSEZ#dG%;TEo)3a=U;p18#jjWwyA)qR&bR%12%pdIb=#l15$wO|S^Ti@_q~a)c;8Ty z|AhLRPQ^vkiv{l=s43^~=eM^0xBPt(^kV(`2k7ZL}5B%293vwC-l^pohoxT}*QBePD=tdy&Z+QQa*MGC~s|5NszPhO( z@gMBGD2RSF^r9sCVW`hv0WS!ouXg_Y{1=tT4_E_|zuEa8U2+ll;u-P-_=xOpz!y&x zH6`?OkpuwXpML=7ttKP|0Kf + for { + archive <- IO(AddonArchive(dummyAddonUrl, "", "")) + path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir) + + aa <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO], path.some) + _ = { + assertEquals(aa.name, "dummy-addon") + assertEquals(aa.version, "2.9") + assertEquals(aa.url, dummyAddonUrl) + } + } yield () + } + + test("Read archive from zip file") { + for { + archive <- AddonArchive.read[IO](dummyAddonUrl, UrlReader.defaultReader[IO]) + _ = { + assertEquals(archive.name, "dummy-addon") + assertEquals(archive.version, "2.9") + assertEquals(archive.url, dummyAddonUrl) + } + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala new file mode 100644 index 00000000..5ad59b14 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.logging.{Level, TestLoggingConfig} + +import munit._ + +class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + + override def docspellLogConfig = + super.docspellLogConfig.copy(minimumLevel = Level.Trace) + + tempDir.test("select docker if Dockerfile exists") { dir => + for { + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker)) + } yield () + } + + tempDir.test("select nix-flake if flake.nix exists") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.NixFlake)) + } yield () + } + + tempDir.test("select nix-flake and docker") { dir => + for { + _ <- files.createFile(dir / "flake.nix") + _ <- files.createFile(dir / "Dockerfile") + cfg = testExecutorConfig( + RunnerType.Docker, + RunnerType.NixFlake, + RunnerType.Trivial + ) + meta = dummyAddonMeta.copy(runner = None) + r <- AddonExecutor.selectRunner[IO](cfg, meta, dir) + _ = assertEquals(r.runnerType, List(RunnerType.Docker, RunnerType.NixFlake)) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala new file mode 100644 index 00000000..9403d02c --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ + +import docspell.common.Glob +import docspell.files.Zip +import docspell.logging.TestLoggingConfig + +import munit._ + +class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures { + val logger = docspell.logging.getLogger[IO] + + test("read meta from zip file") { + val meta = AddonMeta.findInZip(dummyAddonUrl.readURL[IO](8192)) + assertIO(meta, dummyAddonMeta) + } + + tempDir.test("read meta from directory") { dir => + for { + _ <- dummyAddonUrl + .readURL[IO](8192) + .through(Zip.unzip(8192, Glob.all)) + .through(Zip.saveTo(logger, dir, moveUp = true)) + .compile + .drain + meta <- AddonMeta.findInDirectory[IO](dir) + _ = assertEquals(meta, dummyAddonMeta) + } yield () + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala new file mode 100644 index 00000000..0a4bb08e --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonOutputTest.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import docspell.addons.out.AddonOutput + +import io.circe.parser.decode +import munit.FunSuite + +class AddonOutputTest extends FunSuite { + + test("decode empty object") { + val out = decode[AddonOutput]("{}") + println(out) + } + + test("decode sample output") { + val jsonStr = + """{ "files": [ + | { + | "itemId": "qZDnyGIAJsXr", + | "textFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.txt" + | }, + | "pdfFiles": { + | "HPFvIDib6eA": "HPFvIDib6eA.pdf" + | } + | } + | ] + |} + |""".stripMargin + + val out = decode[AddonOutput](jsonStr) + println(out) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala new file mode 100644 index 00000000..1f361ae2 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/AddonRunnerTest.scala @@ -0,0 +1,98 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import java.util.concurrent.atomic.AtomicInteger + +import cats.effect.IO +import fs2.io.file.Path + +import docspell.common.LenientUri +import docspell.common.exec.Env +import docspell.logging.{Logger, TestLoggingConfig} + +import munit._ + +class AddonRunnerTest extends CatsEffectSuite with TestLoggingConfig { + + val logger = docspell.logging.getLogger[IO] + + val dummyContext = Context( + addon = AddonRef(AddonArchive(LenientUri.unsafe("http://test"), "", ""), ""), + meta = AddonMeta.empty("test", "1.0"), + baseDir = Path(""), + addonDir = Path(""), + outputDir = Path(""), + cacheDir = Path("") + ) + + test("firstSuccessful must stop on first success") { + val counter = new AtomicInteger(0) + val runner = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(runner, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("firstSuccessful must try with next on error") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.failWith[IO]("failed") + val runner: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + val r = AddonRunner.firstSuccessful(fail, runner, runner) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 1) + } yield () + } + + test("do not retry on decoding errors") { + val counter = new AtomicInteger(0) + val fail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful(fail, increment, increment) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 0) + } yield () + } + + test("try on errors but stop on decoding error") { + val counter = new AtomicInteger(0) + val decodeFail = AddonRunner.pure[IO](AddonResult.decodingError("Decoding failed")) + val incrementFail = + new MockRunner(IO(counter.incrementAndGet()).void) + .as(AddonResult.executionFailed(new Exception("fail"))) + val increment: AddonRunner[IO] = new MockRunner(IO(counter.incrementAndGet()).void) + + val r = AddonRunner.firstSuccessful( + incrementFail, + incrementFail, + decodeFail, + increment, + increment + ) + for { + _ <- r.run(logger, Env.empty, dummyContext) + _ = assertEquals(counter.get(), 2) + } yield () + } + + final class MockRunner(run: IO[Unit], result: AddonResult = AddonResult.empty) + extends AddonRunner[IO] { + val runnerType = Nil + def run( + logger: Logger[IO], + env: Env, + ctx: Context + ) = run.as(result) + + def as(r: AddonResult) = new MockRunner(run, r) + } +} diff --git a/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala new file mode 100644 index 00000000..c20ac112 --- /dev/null +++ b/modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.addons + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.{Files, Path, PosixPermissions} + +import docspell.addons.AddonExecutorConfig._ +import docspell.addons.AddonMeta._ +import docspell.addons.AddonTriggerType._ +import docspell.common.{Duration, LenientUri} +import docspell.logging.TestLoggingConfig + +import munit.CatsEffectSuite + +trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite => + + val files: Files[IO] = Files[IO] + + val dummyAddonUrl = + LenientUri.fromJava(getClass.getResource("/docspell-dummy-addon-master.zip")) + + val dummyAddonMeta = + AddonMeta( + meta = + AddonMeta.Meta("dummy-addon", "2.9", "Some dummy addon only for testing.\n".some), + triggers = Some( + Set(Scheduled, FinalProcessItem, FinalReprocessItem) + ), + None, + runner = Runner( + nix = NixRunner(true).some, + docker = DockerRunner( + enable = true, + image = None, + build = "Dockerfile".some + ).some, + trivial = TrivialRunner(true, "src/addon.sh").some + ).some, + options = Options(networking = true, collectOutput = true).some + ) + + def baseTempDir: Path = + Path(s"/tmp/target/test-temp") + + val tempDir = + ResourceFixture[Path]( + Resource.eval(Files[IO].createDirectories(baseTempDir)) *> + Files[IO] + .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777")) + ) + + def testExecutorConfig( + runner: RunnerType, + runners: RunnerType* + ): AddonExecutorConfig = { + val nspawn = NSpawn(true, "sudo", "systemd-nspawn", Duration.millis(100)) + AddonExecutorConfig( + runner :: runners.toList, + Duration.minutes(2), + nspawn, + NixConfig("nix", Duration.minutes(2)), + DockerConfig("docker", Duration.minutes(2)) + ) + } +} diff --git a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala index 15ac3f24..9542d3bb 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala @@ -15,8 +15,8 @@ import fs2.io.file.{Files, Path} import docspell.analysis.classifier import docspell.analysis.classifier.TextClassifier._ import docspell.analysis.nlp.Properties -import docspell.common._ import docspell.common.syntax.FileSyntax._ +import docspell.common.util.File import docspell.logging.Logger import edu.stanford.nlp.classify.ColumnDataClassifier diff --git a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala index f8423492..28f9490b 100644 --- a/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala +++ b/modules/analysis/src/main/scala/docspell/analysis/nlp/PipelineCache.scala @@ -14,6 +14,7 @@ import cats.implicits._ import docspell.analysis.NlpSettings import docspell.common._ +import docspell.common.util.File /** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and * initializing large objects. diff --git a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala index 86804836..570a8439 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala @@ -17,6 +17,7 @@ import fs2.io.file.Files import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ +import docspell.common.util.File import docspell.logging.TestLoggingConfig import munit._ diff --git a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala index d522ec6c..15bc0704 100644 --- a/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala +++ b/modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala @@ -13,6 +13,7 @@ import cats.effect.unsafe.implicits.global import docspell.analysis.Env import docspell.common._ +import docspell.common.util.File import docspell.files.TestFiles import docspell.logging.TestLoggingConfig diff --git a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala index 015e771b..386e2a4a 100644 --- a/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala +++ b/modules/backend/src/main/scala/docspell/backend/AttachedEvent.scala @@ -20,6 +20,7 @@ trait AttachedEvent[R] { object AttachedEvent { + /** Only the result, no events. */ def only[R](v: R): AttachedEvent[R] = new AttachedEvent[R] { val value = v diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index a34eaf6a..cc55c72b 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -8,11 +8,14 @@ package docspell.backend import cats.effect._ +import docspell.backend.BackendCommands.EventContext import docspell.backend.auth.Login import docspell.backend.fulltext.CreateIndex import docspell.backend.ops._ import docspell.backend.signup.OSignup +import docspell.common.bc.BackendCommandRunner import docspell.ftsclient.FtsClient +import docspell.joexapi.client.JoexClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT import docspell.scheduler.JobStoreModule @@ -20,6 +23,7 @@ import docspell.store.Store import docspell.totp.Totp import emil.Emil +import org.http4s.client.Client trait BackendApp[F[_]] { @@ -35,6 +39,7 @@ trait BackendApp[F[_]] { def job: OJob[F] def item: OItem[F] def itemSearch: OItemSearch[F] + def attachment: OAttachment[F] def fulltext: OFulltext[F] def mail: OMail[F] def joex: OJoex[F] @@ -52,23 +57,30 @@ trait BackendApp[F[_]] { def fileRepository: OFileRepository[F] def itemLink: OItemLink[F] def downloadAll: ODownloadAll[F] + def addons: OAddons[F] + + def commands(eventContext: Option[EventContext]): BackendCommandRunner[F, Unit] } object BackendApp { def create[F[_]: Async]( + cfg: Config, store: Store[F], javaEmil: Emil[F], + httpClient: Client[F], ftsClient: FtsClient[F], pubSubT: PubSubT[F], schedulerModule: JobStoreModule[F], notificationMod: NotificationModule[F] ): Resource[F, BackendApp[F]] = for { + nodeImpl <- ONode(store) totpImpl <- OTotp(store, Totp.default) loginImpl <- Login[F](store, Totp.default) signupImpl <- OSignup[F](store) - joexImpl <- OJoex(pubSubT) + joexClient = JoexClient(httpClient) + joexImpl <- OJoex(pubSubT, nodeImpl, joexClient) collImpl <- OCollective[F]( store, schedulerModule.userTasks, @@ -80,7 +92,6 @@ object BackendApp { equipImpl <- OEquipment[F](store) orgImpl <- OOrganization(store) uploadImpl <- OUpload(store, schedulerModule.jobs) - nodeImpl <- ONode(store) jobImpl <- OJob(store, joexImpl, pubSubT) createIndex <- CreateIndex.resource(ftsClient, store) itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs) @@ -109,6 +120,16 @@ object BackendApp { fileRepoImpl <- OFileRepository(store, schedulerModule.jobs) itemLinkImpl <- Resource.pure(OItemLink(store, itemSearchImpl)) downloadAllImpl <- Resource.pure(ODownloadAll(store, jobImpl, schedulerModule.jobs)) + attachImpl <- Resource.pure(OAttachment(store, ftsClient, schedulerModule.jobs)) + addonsImpl <- Resource.pure( + OAddons( + cfg.addons, + store, + schedulerModule.userTasks, + schedulerModule.jobs, + joexImpl + ) + ) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl @@ -139,5 +160,10 @@ object BackendApp { val fileRepository = fileRepoImpl val itemLink = itemLinkImpl val downloadAll = downloadAllImpl + val addons = addonsImpl + val attachment = attachImpl + + def commands(eventContext: Option[EventContext]) = + BackendCommands.fromBackend(this, eventContext) } } diff --git a/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala new file mode 100644 index 00000000..3550cfbf --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/BackendCommands.scala @@ -0,0 +1,175 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ + +import docspell.backend.BackendCommands.EventContext +import docspell.backend.ops.OCustomFields.SetValue +import docspell.backend.ops._ +import docspell.common.bc._ +import docspell.common.{AccountId, Ident, LenientUri} + +private[backend] class BackendCommands[F[_]: Sync]( + itemOps: OItem[F], + attachOps: OAttachment[F], + fieldOps: OCustomFields[F], + notificationOps: ONotification[F], + eventContext: Option[EventContext] +) extends BackendCommandRunner[F, Unit] { + private[this] val logger = docspell.logging.getLogger[F] + + def run(collective: Ident, cmd: BackendCommand): F[Unit] = + doRun(collective, cmd).attempt.flatMap { + case Right(_) => ().pure[F] + case Left(ex) => + logger.error(ex)(s"Backend command $cmd failed for collective ${collective.id}.") + } + + def doRun(collective: Ident, cmd: BackendCommand): F[Unit] = + cmd match { + case BackendCommand.ItemUpdate(item, actions) => + actions.traverse_(a => runItemAction(collective, item, a)) + + case BackendCommand.AttachmentUpdate(item, attach, actions) => + actions.traverse_(a => runAttachAction(collective, item, attach, a)) + } + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[Unit] = + cmds.traverse_(run(collective, _)) + + def runItemAction(collective: Ident, item: Ident, action: ItemAction): F[Unit] = + action match { + case ItemAction.AddTags(tags) => + logger.debug(s"Setting tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .linkTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.RemoveTags(tags) => + logger.debug(s"Remove tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .removeTagsMultipleItems(Nel.of(item), tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.ReplaceTags(tags) => + logger.debug(s"Replace tags $tags on ${item.id} for ${collective.id}") *> + itemOps + .setTags(item, tags.toList, collective) + .flatMap(sendEvents) + + case ItemAction.SetFolder(folder) => + logger.debug(s"Set folder $folder on ${item.id} for ${collective.id}") *> + itemOps + .setFolder(item, folder, collective) + .void + + case ItemAction.RemoveTagsCategory(cats) => + logger.debug( + s"Remove tags in categories $cats on ${item.id} for ${collective.id}" + ) *> + itemOps + .removeTagsOfCategories(item, collective, cats) + .flatMap(sendEvents) + + case ItemAction.SetCorrOrg(id) => + logger.debug( + s"Set correspondent organization ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrOrg(Nel.of(item), id, collective).void + + case ItemAction.SetCorrPerson(id) => + logger.debug( + s"Set correspondent person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setCorrPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcPerson(id) => + logger.debug( + s"Set concerning person ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcPerson(Nel.of(item), id, collective).void + + case ItemAction.SetConcEquipment(id) => + logger.debug( + s"Set concerning equipment ${id.map(_.id)} for ${collective.id}" + ) *> + itemOps.setConcEquip(Nel.of(item), id, collective).void + + case ItemAction.SetField(field, value) => + logger.debug( + s"Set field on item ${item.id} ${field.id} to '$value' for ${collective.id}" + ) *> + fieldOps + .setValue(item, SetValue(field, value, collective)) + .flatMap(sendEvents) + + case ItemAction.SetNotes(notes) => + logger.debug(s"Set notes on item ${item.id} for ${collective.id}") *> + itemOps.setNotes(item, notes, collective).void + + case ItemAction.AddNotes(notes, sep) => + logger.debug(s"Add notes on item ${item.id} for ${collective.id}") *> + itemOps.addNotes(item, notes, sep, collective).void + + case ItemAction.SetName(name) => + logger.debug(s"Set name '$name' on item ${item.id} for ${collective.id}") *> + itemOps.setName(item, name, collective).void + } + + def runAttachAction( + collective: Ident, + itemId: Ident, + attachId: Ident, + action: AttachmentAction + ): F[Unit] = + action match { + case AttachmentAction.SetExtractedText(text) => + attachOps.setExtractedText( + collective, + itemId, + attachId, + text.getOrElse("").pure[F] + ) + } + + private def sendEvents(result: AttachedEvent[_]): F[Unit] = + eventContext match { + case Some(ctx) => + notificationOps.offerEvents(result.event(ctx.account, ctx.baseUrl)) + case None => ().pure[F] + } +} + +object BackendCommands { + + /** If supplied, notification events will be send. */ + case class EventContext(account: AccountId, baseUrl: Option[LenientUri]) + + def fromBackend[F[_]: Sync]( + backendApp: BackendApp[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F]( + backendApp.item, + backendApp.attachment, + backendApp.customFields, + backendApp.notification, + eventContext + ) + + def apply[F[_]: Sync]( + item: OItem[F], + attachment: OAttachment[F], + fields: OCustomFields[F], + notification: ONotification[F], + eventContext: Option[EventContext] = None + ): BackendCommandRunner[F, Unit] = + new BackendCommands[F](item, attachment, fields, notification, eventContext) +} diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index efccb8dc..b1baedcb 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -20,7 +20,8 @@ case class Config( mailDebug: Boolean, jdbc: JdbcConfig, signup: SignupConfig, - files: Config.Files + files: Config.Files, + addons: Config.Addons ) { def mailSettings: Settings = @@ -66,4 +67,21 @@ object Config { (storesEmpty |+| defaultStorePresent).map(_ => this) } } + + case class Addons( + enabled: Boolean, + allowImpure: Boolean, + allowedUrls: UrlMatcher, + deniedUrls: UrlMatcher + ) { + def isAllowed(url: LenientUri): Boolean = + allowedUrls.matches(url) && !deniedUrls.matches(url) + + def isDenied(url: LenientUri): Boolean = + !isAllowed(url) + } + object Addons { + val disabled: Addons = + Addons(false, false, UrlMatcher.False, UrlMatcher.True) + } } diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index 87adbbdc..cf87d783 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -16,6 +16,26 @@ import docspell.notification.api.PeriodicQueryArgs import docspell.scheduler.Job object JobFactory extends MailAddressCodec { + def existingItemAddon[F[_]: Sync]( + args: ItemAddonTaskArgs, + submitter: AccountId + ): F[Job[ItemAddonTaskArgs]] = + Job.createNew( + ItemAddonTaskArgs.taskName, + submitter.collective, + args, + "Run addons on item", + submitter.user, + Priority.High, + args.addonRunConfigs + .map(_.take(23)) + .toList + .sorted + .foldLeft(args.itemId)(_ / _) + .take(250) + .some + ) + def downloadZip[F[_]: Sync]( args: DownloadZipArgs, summaryId: Ident, diff --git a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala index a30e0922..fd5f9214 100644 --- a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala +++ b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala @@ -45,7 +45,14 @@ object CreateIndex { chunkSize: Int ): F[Unit] = { val attachs = store - .transact(QAttachment.allAttachmentMetaAndName(collective, itemIds, chunkSize)) + .transact( + QAttachment.allAttachmentMetaAndName( + collective, + itemIds, + ItemState.validStates, + chunkSize + ) + ) .map(caa => TextData .attachment( diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala new file mode 100644 index 00000000..dba1cbf4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonEnvConfig.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import fs2.io.file.Path + +import docspell.addons.AddonExecutorConfig + +final case class AddonEnvConfig( + workingDir: Path, + cacheDir: Path, + executorConfig: AddonExecutorConfig +) diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala new file mode 100644 index 00000000..f69dd2a7 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala @@ -0,0 +1,199 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons._ +import docspell.backend.joex.AddonOps.{AddonRunConfigRef, ExecResult} +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.common.exec.Env +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records.AddonRunConfigResolved + +trait AddonOps[F[_]] { + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + middleware: Middleware[F] + ): F[ExecResult] + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + middleware: Middleware[F] + ): F[ExecResult] + + /** Find enabled addon run config references to be executed. Can be additionally + * filtered by given ids and triggers. + */ + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] + + /** Find enabled addon run config reference given an addon task id */ + def findAddonRef(collective: Ident, runConfigId: Ident): F[Option[AddonRunConfigRef]] + + /** Creates an executor for addons given a configuration. */ + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] + +} + +object AddonOps { + case class AddonRunConfigRef( + id: Ident, + collective: Ident, + userId: Option[Ident], + name: String, + refs: List[AddonRef] + ) + + object AddonRunConfigRef { + def fromResolved(r: AddonRunConfigResolved): AddonRunConfigRef = + AddonRunConfigRef( + r.config.id, + r.config.cid, + r.config.userId, + r.config.name, + r.refs.map(ref => AddonRef(ref.archive.asArchive, ref.ref.args)) + ) + } + + case class ExecResult( + result: List[AddonExecutionResult], + runConfigs: List[AddonRunConfigRef] + ) { + lazy val combined = result.combineAll + } + + object ExecResult { + def runConfigNotFound(id: Ident): ExecResult = + ExecResult( + AddonExecutionResult( + AddonResult.executionFailed( + new Exception(s"Addon run config ${id.id} not found.") + ) :: Nil, + false + ) :: Nil, + Nil + ) + } + + def apply[F[_]: Async]( + cfg: AddonEnvConfig, + store: Store[F], + cmdRunner: BackendCommandRunner[F, Unit], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonOps[F] = + new AddonOps[F] with LoggerExtension { + private[this] val logger = docspell.logging.getLogger[F] + + private val urlReader = FileUrlReader(store.fileRepo) + private val postProcess = AddonPostProcess(cmdRunner, store, attachment, jobStore) + private val prepare = new AddonPrepare[F](store) + + def execAll( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident], + logger: Option[Logger[F]] + )( + custom: Middleware[F] + ): F[ExecResult] = + for { + runCfgs <- findAddonRefs(collective, trigger, runConfigIds) + log = logger.getOrElse(this.logger) + _ <- log.info(s"Running ${runCfgs.size} addon tasks for trigger $trigger") + + results <- runCfgs.traverse(r => execRunConfig(log, r, custom)) + } yield ExecResult(results.flatMap(_.result), runCfgs) + + def execById(collective: Ident, runConfigId: Ident, logger: Logger[F])( + custom: Middleware[F] + ): F[ExecResult] = + (for { + runCfg <- OptionT(findAddonRef(collective, runConfigId)) + execRes <- OptionT.liftF(execRunConfig(logger, runCfg, custom)) + } yield execRes).getOrElse(ExecResult.runConfigNotFound(runConfigId)) + + def execRunConfig( + logger: Logger[F], + runCfg: AddonRunConfigRef, + custom: Middleware[F] + ): F[ExecResult] = + for { + executor <- getExecutor(cfg.executorConfig) + log = logger.withRunConfig(runCfg) + result <- + Directory.temp(cfg.workingDir, "addon-output-").use { outDir => + val cacheDir = cfg.cacheDir / runCfg.id.id + val inputEnv = + InputEnv(runCfg.refs, cfg.workingDir, outDir, cacheDir, Env.empty) + + for { + middleware <- createMiddleware(custom, runCfg) + res <- middleware(executor.execute(log)).run(inputEnv) + _ <- log.debug(s"Addon result: $res") + _ <- postProcess.onResult(log, runCfg.collective, res, outDir) + } yield res + } + execRes = ExecResult(List(result), List(runCfg)) + } yield execRes + + def createMiddleware(custom: Middleware[F], runCfg: AddonRunConfigRef) = for { + dscMW <- prepare.createDscEnv(runCfg, cfg.executorConfig.runTimeout) + mm = dscMW >> custom >> prepare.logResult(logger, runCfg) >> Middleware + .ephemeralRun[F] + } yield mm + + def getExecutor(cfg: AddonExecutorConfig): F[AddonExecutor[F]] = + Async[F].pure(AddonExecutor(cfg, urlReader)) + + def findAddonRefs( + collective: Ident, + trigger: Set[AddonTriggerType], + runConfigIds: Set[Ident] + ): F[List[AddonRunConfigRef]] = + store + .transact( + AddonRunConfigResolved.findAllForCollective( + collective, + enabled = true.some, + trigger, + runConfigIds + ) + ) + .map(_.map(AddonRunConfigRef.fromResolved)) + + def findAddonRef( + collective: Ident, + runConfigId: Ident + ): F[Option[AddonRunConfigRef]] = + OptionT( + store + .transact( + AddonRunConfigResolved.findById( + runConfigId, + collective, + enabled = Some(true) + ) + ) + ).map(AddonRunConfigRef.fromResolved).value + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala new file mode 100644 index 00000000..2696b4e9 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPostProcess.scala @@ -0,0 +1,198 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.OptionT +import cats.effect.kernel.Sync +import cats.syntax.all._ +import fs2.io.file.{Files, Path} + +import docspell.addons._ +import docspell.addons.out.{AddonOutput, ItemFile, NewItem} +import docspell.backend.JobFactory +import docspell.backend.ops.OAttachment +import docspell.common._ +import docspell.common.bc.BackendCommandRunner +import docspell.files.FileSupport +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.records._ + +final private[joex] class AddonPostProcess[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachOps: OAttachment[F], + jobStore: JobStore[F] +) extends FileSupport { + + def onResult( + logger: Logger[F], + collective: Ident, + result: AddonExecutionResult, + outputDir: Path + ): F[Unit] = + result.addonResult match { + case AddonResult.Success(output) => + onSuccess(logger, collective, output, outputDir) + case _ => + ().pure[F] + } + + def onSuccess( + logger: Logger[F], + collective: Ident, + output: AddonOutput, + outputDir: Path + ): F[Unit] = + for { + _ <- logger.info("Applying addon output") + _ <- cmdRunner.runAll(collective, output.commands) + _ <- logger.debug("Applying changes from files") + _ <- output.files.traverse_(updateOne(logger, collective, outputDir)) + _ <- output.newItems.traverse_(submitNewItem(logger, collective, outputDir)) + } yield () + + def submitNewItem( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(newItem: NewItem): F[Unit] = + for { + _ <- logger.info(s"Submit new item with ${newItem.files.size} files") + files <- newItem.resolveFiles[F](logger, outputDir) + collLang <- store.transact(RCollective.findLanguage(collective)) + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + _ <- logger.debug(s"Saved ${uploaded.size} files to be processed.") + args = ProcessItemArgs( + newItem.toProcessMeta(collective, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + account = AccountId(collective, DocspellSystem.user) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + + def updateOne(logger: Logger[F], collective: Ident, outputDir: Path)( + itemFile: ItemFile + ): F[Unit] = + for { + textFiles <- itemFile.resolveTextFiles(logger, outputDir) + pdfFiles <- itemFile.resolvePdfFiles(logger, outputDir) + previewFiles <- itemFile.resolvePreviewFiles(logger, outputDir) + attachs <- OptionT + .whenF(textFiles.nonEmpty || pdfFiles.nonEmpty || previewFiles.nonEmpty)( + store.transact(RAttachment.findByItem(itemFile.itemId)) + ) + .getOrElse(Vector.empty) + _ <- textFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + setText(collective, ra, file.readText) + } + } + _ <- pdfFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePdf(collective, ra, file, previewFiles.forall(_._1 != key)) + } + } + _ <- previewFiles.traverse_ { case (key, file) => + withAttach(logger, key, attachs) { ra => + replacePreview(collective, ra.id, file) + } + } + _ <- submitNewFiles(logger, collective, outputDir)(itemFile) + } yield () + + def submitNewFiles( + logger: Logger[F], + collective: Ident, + outputDir: Path + )(itemFile: ItemFile): F[Unit] = + for { + _ <- logger.info(s"Submitting new file for item") + collLang <- store.transact(RCollective.findLanguage(collective)) + newFiles <- itemFile.resolveNewFiles(logger, outputDir) + byMeta = newFiles.groupBy(_._1.metadata).view.mapValues(_.map(_._2)) + account = AccountId(collective, DocspellSystem.user) + _ <- byMeta.toList.traverse_ { case (meta, files) => + for { + uploaded <- files.traverse(file => + file.readAll + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentSource, + MimeTypeHint.filename(file) + ) + ) + .compile + .lastOrError + .map(key => file.fileName.toString -> key) + ) + args = ProcessItemArgs( + meta.toProcessMeta(collective, itemFile.itemId, collLang, "addon"), + uploaded.map(f => ProcessItemArgs.File(f._1.some, f._2)) + ) + job <- JobFactory.processItem(args, account, Priority.High, None) + _ <- jobStore.insert(job.encode) + _ <- logger.debug(s"Submitted job for processing: ${job.id}") + } yield () + } + } yield () + + private def withAttach(logger: Logger[F], key: String, attachs: Vector[RAttachment])( + run: RAttachment => F[Unit] + ): F[Unit] = + OptionT + .fromOption( + attachs.find(a => a.id.id == key || key.toIntOption == a.position.some) + ) + .semiflatMap(run) + .getOrElseF(logger.warn(s"Cannot find attachment for $key to update text!")) + + private def setText(collective: Ident, ra: RAttachment, readText: F[String]): F[Unit] = + attachOps.setExtractedText(collective, ra.itemId, ra.id, readText) + + private def replacePdf( + collective: Ident, + ra: RAttachment, + file: Path, + generatePreview: Boolean + ): F[Unit] = + attachOps.addOrReplacePdf(collective, ra.id, file.readAll, generatePreview) + + private def replacePreview( + collective: Ident, + attachId: Ident, + imageData: Path + ): F[Unit] = + attachOps.addOrReplacePreview(collective, attachId, imageData.readAll) +} + +object AddonPostProcess { + + def apply[F[_]: Sync: Files]( + cmdRunner: BackendCommandRunner[F, Unit], + store: Store[F], + attachment: OAttachment[F], + jobStore: JobStore[F] + ): AddonPostProcess[F] = + new AddonPostProcess[F](cmdRunner, store, attachment, jobStore) +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala new file mode 100644 index 00000000..03ab223d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.auth.AuthToken +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.common._ +import docspell.logging.Logger +import docspell.store.Store +import docspell.store.records.{RNode, RUser} + +import scodec.bits.ByteVector + +private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExtension { + + def logResult(logger: Logger[F], ref: AddonRunConfigRef): Middleware[F] = + Middleware(_.mapF(_.attempt.flatTap { + case Right(_) => ().pure[F] + case Left(ex) => + logger + .withRunConfig(ref) + .warn(ex)(s"Addon task '${ref.id.id}' has failed") + }.rethrow)) + + /** Creates environment variables for dsc to connect to the docspell server for the + * given run config. + */ + def createDscEnv( + runConfigRef: AddonRunConfigRef, + tokenValidity: Duration + ): F[Middleware[F]] = + (for { + userId <- OptionT.fromOption[F](runConfigRef.userId) + user <- OptionT(store.transact(RUser.getIdByIdOrLogin(userId))) + account = AccountId(runConfigRef.collective, user.login) + env = + Middleware.prepare[F]( + Kleisli(input => makeDscEnv(account, tokenValidity).map(input.addEnv)) + ) + } yield env).getOrElse(Middleware.identity[F]) + + /** Creates environment variables to have dsc automatically connect as the given user. + * Additionally a random rest-server is looked up from the database to set its url. + */ + def makeDscEnv( + accountId: AccountId, + tokenValidity: Duration + ): F[Map[String, String]] = + for { + serverNode <- store.transact( + RNode + .findAll(NodeType.Restserver) + .map(_.sortBy(_.updated).lastOption) + ) + url = serverNode.map(_.url).map(u => "DSC_DOCSPELL_URL" -> u.asString) + secret = serverNode.flatMap(_.serverSecret) + + token <- AuthToken.user( + accountId, + false, + secret.getOrElse(ByteVector.empty), + tokenValidity.some + ) + session = ("DSC_SESSION" -> token.asString).some + } yield List(url, session).flatten.toMap +} diff --git a/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala new file mode 100644 index 00000000..1f3e4da1 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/joex/LoggerExtension.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.joex + +import docspell.backend.joex.AddonOps.AddonRunConfigRef +import docspell.logging.Logger + +trait LoggerExtension { + + implicit final class LoggerDataOps[F[_]](self: Logger[F]) { + def withRunConfig(t: AddonRunConfigRef): Logger[F] = + self.capture("addon-task-id", t.id) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala new file mode 100644 index 00000000..e634233a --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigError.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} + +sealed trait AddonRunConfigError { + final def cast: AddonRunConfigError = this + + def toLeft[A]: Either[AddonRunConfigError, A] = Left(this) + + def message: String +} + +object AddonRunConfigError { + + case object MissingSchedule extends AddonRunConfigError { + val message = + "The run config has a trigger 'scheduled' but doesn't provide a schedule!" + } + + case object ObsoleteSchedule extends AddonRunConfigError { + val message = "The run config has a schedule, but not a trigger 'Scheduled'." + } + + case class MismatchingTrigger(unsupported: NonEmptyList[(String, AddonTriggerType)]) + extends AddonRunConfigError { + def message: String = { + val list = + unsupported.map { case (name, tt) => s"$name: ${tt.name}" }.toList.mkString(", ") + s"Some listed addons don't support all defined triggers: $list" + } + } + + object MismatchingTrigger { + def apply(addon: AddonMeta, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + + def apply(addon: AddonArchive, tt: AddonTriggerType): MismatchingTrigger = + MismatchingTrigger(NonEmptyList.of(addon.nameAndVersion -> tt)) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala new file mode 100644 index 00000000..c675c13d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonRunConfigValidate.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.ops.AddonRunConfigError._ +import docspell.backend.ops.OAddons.{AddonRunConfigResult, AddonRunInsert} +import docspell.common.Ident +import docspell.store.Store +import docspell.store.records.RAddonArchive + +object AddonRunConfigValidate { + + def apply[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[AddonRunInsert]] = { + val init: AddonRunConfigResult[Unit] = ().asRight + + List( + checkScheduled(cfg).pure[F], + checkTriggers(store, cid)(cfg) + ) + .foldLeftM(init)((res, fr) => fr.map(r => res.flatMap(_ => r))) + .map(_.as(cfg)) + } + + def checkTriggers[F[_]: Sync](store: Store[F], cid: Ident)( + cfg: AddonRunInsert + ): F[AddonRunConfigResult[Unit]] = + for { + addons <- store.transact(RAddonArchive.findByIds(cid, cfg.addons.map(_.addonId))) + given = cfg.triggered.toList.toSet + res = addons + .flatMap(r => given.diff(r.triggers).map(tt => r.nameAndVersion -> tt)) + + maybeError = NonEmptyList + .fromList(res) + .map(nel => MismatchingTrigger(nel)) + } yield maybeError.map(_.toLeft).getOrElse(Right(())) + + def checkScheduled(cfg: AddonRunInsert): AddonRunConfigResult[Unit] = + (cfg.isScheduled, cfg.schedule) match { + case (true, None) => MissingSchedule.toLeft[Unit] + case (false, Some(_)) => ObsoleteSchedule.toLeft[Unit] + case _ => ().asRight + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala new file mode 100644 index 00000000..6074f557 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidate.scala @@ -0,0 +1,156 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.EitherT +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.Path + +import docspell.addons.{AddonMeta, RunnerType} +import docspell.backend.Config +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.{Ident, LenientUri, UrlReader} +import docspell.joexapi.model.AddonSupport +import docspell.store.Store +import docspell.store.records.RAddonArchive + +final class AddonValidate[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + joexOps: OJoex[F] +) { + private[this] val logger = docspell.logging.getLogger[F] + + def fromUrl( + collective: Ident, + url: LenientUri, + reader: UrlReader[F], + localUrl: Option[LenientUri] = None, + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + if (!cfg.enabled) AddonsDisabled.resultF + else if (cfg.isDenied(url)) UrlUntrusted(url).resultF + else if (checkExisting) + store.transact(RAddonArchive.findByUrl(collective, url)).flatMap { + case Some(ar) => + AddonExists("An addon with this url already exists!", ar).resultF + case None => + archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + } + else archive(collective, reader(localUrl.getOrElse(url)).asRight, checkExisting) + + def archive( + collective: Ident, + addonData: Either[Path, Stream[F, Byte]], + checkExisting: Boolean = true + ): F[AddonValidationResult[AddonMeta]] = + (for { + _ <- EitherT.cond[F](cfg.enabled, (), AddonsDisabled.cast) + + meta <- + EitherT( + addonData + .fold( + AddonMeta.findInDirectory[F], + AddonMeta.findInZip[F] + ) + .attempt + ) + .leftMap(ex => NotAnAddon(ex).cast) + _ <- EitherT.cond( + meta.triggers.exists(_.nonEmpty), + (), + InvalidAddon( + "The addon doesn't define any triggers. At least one is required!" + ).cast + ) + _ <- EitherT.cond( + meta.options.exists(_.isUseful), + (), + InvalidAddon( + "Addon defines no output and no networking. It can't do anything useful." + ).cast + ) + _ <- EitherT.cond(cfg.allowImpure || meta.isPure, (), ImpureAddonsDisabled.cast) + + _ <- + if (checkExisting) + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(collective, meta.meta.name, meta.meta.version) + ) + .map { + case Some(ar) => AddonExists(ar).result + case None => rightUnit + } + ) + else rightUnitT + + joexSupport <- EitherT.liftF(joexOps.getAddonSupport) + addonRunners <- EitherT.liftF(meta.enabledTypes(addonData)) + _ <- EitherT.liftF( + logger.info( + s"Comparing joex support vs addon runner: $joexSupport vs. $addonRunners" + ) + ) + _ <- EitherT.fromEither(validateJoexSupport(addonRunners, joexSupport)) + + } yield meta).value + + private def validateJoexSupport( + addonRunnerTypes: List[RunnerType], + joexSupport: List[AddonSupport] + ): AddonValidationResult[Unit] = { + val addonRunners = addonRunnerTypes.mkString(", ") + for { + _ <- Either.cond( + joexSupport.nonEmpty, + (), + AddonUnsupported("There are no joex nodes that have addons enabled!", Nil).cast + ) + _ <- Either.cond( + addonRunners.nonEmpty, + (), + InvalidAddon("The addon doesn't enable any runner.") + ) + + ids = joexSupport + .map(n => n.nodeId -> n.runners.intersect(addonRunnerTypes).toSet) + + unsupportedJoex = ids.filter(_._2.isEmpty).map(_._1) + + _ <- Either.cond( + ids.forall(_._2.nonEmpty), + (), + AddonUnsupported( + s"A joex node doesn't support this addons runners: $addonRunners. " + + s"Check: ${unsupportedJoex.map(_.id).mkString(", ")}.", + unsupportedJoex + ).cast + ) + } yield () + } + + private def rightUnit: AddonValidationResult[Unit] = + ().asRight[AddonValidationError] + + private def rightUnitT: EitherT[F, AddonValidationError, Unit] = + EitherT.fromEither(rightUnit) + + implicit final class ErrorOps(self: AddonValidationError) { + def result: AddonValidationResult[AddonMeta] = + self.toLeft + + def resultF: F[AddonValidationResult[AddonMeta]] = + result.pure[F] + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala new file mode 100644 index 00000000..d8f77e24 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/AddonValidationError.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import docspell.common.{Ident, LenientUri} +import docspell.store.records.RAddonArchive + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AddonValidationError { + def cast: AddonValidationError = this + + def toLeft[A]: Either[AddonValidationError, A] = Left(this) +} + +object AddonValidationError { + + implicit private val throwableDecoder: Decoder[Throwable] = + Decoder.decodeString.map(new Exception(_)) + implicit private val throwableEncoder: Encoder[Throwable] = + Encoder.encodeString.contramap(_.getMessage) + + case object AddonsDisabled extends AddonValidationError {} + + case class UrlUntrusted(url: LenientUri) extends AddonValidationError + object UrlUntrusted { + implicit val jsonDecoder: Decoder[UrlUntrusted] = deriveDecoder + implicit val jsonEncoder: Encoder[UrlUntrusted] = deriveEncoder + } + + case class NotAnAddon(error: Throwable) extends AddonValidationError + object NotAnAddon { + implicit val jsonDecoder: Decoder[NotAnAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[NotAnAddon] = deriveEncoder + } + + case class AddonUnsupported(message: String, affectedNodes: List[Ident]) + extends AddonValidationError + object AddonUnsupported { + implicit val jsonDecoder: Decoder[AddonUnsupported] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonUnsupported] = deriveEncoder + } + + case class InvalidAddon(message: String) extends AddonValidationError + object InvalidAddon { + implicit val jsonDecoder: Decoder[InvalidAddon] = deriveDecoder + implicit val jsonEncoder: Encoder[InvalidAddon] = deriveEncoder + } + + case class AddonExists(message: String, addon: RAddonArchive) + extends AddonValidationError + object AddonExists { + def apply(addon: RAddonArchive): AddonExists = + AddonExists(s"An addon '${addon.name}/${addon.version}' already exists!", addon) + + implicit val jsonDecoder: Decoder[AddonExists] = deriveDecoder + implicit val jsonEncoder: Encoder[AddonExists] = deriveEncoder + } + + case object AddonNotFound extends AddonValidationError + + case class DownloadFailed(error: Throwable) extends AddonValidationError + object DownloadFailed { + implicit val jsonDecoder: Decoder[DownloadFailed] = deriveDecoder + implicit val jsonEncoder: Encoder[DownloadFailed] = deriveEncoder + } + + case object ImpureAddonsDisabled extends AddonValidationError + + case object RefreshLocalAddon extends AddonValidationError + + implicit val jsonConfig: Configuration = + Configuration.default.withKebabCaseConstructorNames + .withDiscriminator("errorType") + + implicit val jsonDecoder: Decoder[AddonValidationError] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AddonValidationError] = deriveConfiguredEncoder +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala new file mode 100644 index 00000000..be5232a4 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAddons.scala @@ -0,0 +1,426 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{EitherT, NonEmptyList, OptionT} +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.{AddonMeta, AddonTriggerType} +import docspell.backend.ops.AddonValidationError._ +import docspell.backend.ops.OAddons._ +import docspell.backend.{Config, JobFactory} +import docspell.common._ +import docspell.logging.Logger +import docspell.scheduler.JobStore +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} +import docspell.store.Store +import docspell.store.file.FileUrlReader +import docspell.store.records._ + +import com.github.eikek.calev.CalEvent + +trait OAddons[F[_]] { + + /** Registers a new addon. An error is returned if an addon with this url already + * exists. + */ + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Refreshes an existing addon by downloading it again and updating metadata. */ + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] + + /** Look into the addon at the given url and return its metadata. */ + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] + + /** Deletes the addon if it exists. */ + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] + + /** Inserts or updates the addon run configuration. If it already exists (and the given + * id is non empty), it will be completely replaced with the given one. + */ + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] + + /** Deletes this task from the database. */ + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] +} + +object OAddons { + val scheduledAddonTaskName: Ident = + ScheduledAddonTaskArgs.taskName + + case class AddonRunInsert( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: NonEmptyList[AddonTriggerType], + addons: NonEmptyList[AddonArgs] + ) { + + def isScheduled: Boolean = + triggered.exists(_ == AddonTriggerType.Scheduled) + } + case class AddonArgs(addonId: Ident, args: String) + + case class AddonRunInfo( + id: Ident, + name: String, + enabled: Boolean, + userId: Option[Ident], + schedule: Option[CalEvent], + triggered: List[AddonTriggerType], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + ) + object AddonRunInfo { + def fromRunConfigData( + timer: Option[CalEvent], + addons: List[(RAddonArchive, RAddonRunConfigAddon)] + )(t: AddonRunConfigData): AddonRunInfo = + AddonRunInfo( + id = t.runConfig.id, + name = t.runConfig.name, + enabled = t.runConfig.enabled, + userId = t.runConfig.userId, + schedule = timer, + triggered = t.triggers.map(_.trigger), + addons = addons + ) + } + + type AddonRunConfigResult[A] = Either[AddonRunConfigError, A] + object AddonRunConfigResult { + def success[A](value: A): AddonRunConfigResult[A] = Right(value) + def failure[A](error: AddonRunConfigError): AddonRunConfigResult[A] = error.toLeft[A] + } + + type AddonValidationResult[A] = Either[AddonValidationError, A] + object AddonValidationResult { + def success[A](value: A): AddonValidationResult[A] = Right(value) + def failure[A](error: AddonValidationError): AddonValidationResult[A] = Left(error) + } + + def apply[F[_]: Async]( + cfg: Config.Addons, + store: Store[F], + userTasks: UserTaskStore[F], + jobStore: JobStore[F], + joex: OJoex[F] + ): OAddons[F] = + new OAddons[F] { + private[this] val logger = docspell.logging.getLogger[F] + private val urlReader = FileUrlReader(store.fileRepo) + private val zip = MimeType.zip.asString + private val addonValidate = new AddonValidate[F](cfg, store, joex) + + def getAllAddonRunConfigs(collective: Ident): F[List[AddonRunInfo]] = + for { + all <- store.transact(AddonRunConfigData.findAll(collective)) + runConfigIDs = all.map(_.runConfig.id).toSet + archiveIds = all.flatMap(_.addons.map(_.addonId)).distinct + archives <- NonEmptyList + .fromList(archiveIds) + .fold(List.empty[RAddonArchive].pure[F])(ids => + store.transact(RAddonArchive.findByIds(collective, ids)) + ) + archivesMap = archives.groupBy(_.id) + ptask <- userTasks + .getAll(UserTaskScope.collective(collective)) + .filter(ut => runConfigIDs.contains(ut.id)) + .map(ut => ut.id -> ut) + .compile + .toList + .map(_.toMap) + result = all.map { t => + AddonRunInfo.fromRunConfigData( + ptask.get(t.runConfig.id).map(_.timer), + t.addons.map(raa => (archivesMap(raa.addonId).head, raa)) + )(t) + } + } yield result + + def upsertAddonRunConfig( + collective: Ident, + runConfig: AddonRunInsert + ): F[AddonRunConfigResult[Ident]] = { + val insertDataRaw = AddonRunConfigData( + RAddonRunConfig( + runConfig.id, + collective, + runConfig.userId, + runConfig.name, + runConfig.enabled, + Timestamp.Epoch + ), + runConfig.addons.zipWithIndex.map { case (a, index) => + RAddonRunConfigAddon(Ident.unsafe(""), runConfig.id, a.addonId, a.args, index) + }.toList, + runConfig.triggered + .map(t => RAddonRunConfigTrigger(Ident.unsafe(""), runConfig.id, t)) + .toList + ) + + val upsert = for { + userId <- + OptionT + .fromOption(runConfig.userId) + .flatMapF(uid => store.transact(RUser.getIdByIdOrLogin(uid))) + .map(_.uid) + .value + insertData = + insertDataRaw.copy(runConfig = + insertDataRaw.runConfig.copy(userId = userId.orElse(runConfig.userId)) + ) + id <- + OptionT(store.transact(RAddonRunConfig.findById(collective, runConfig.id))) + .map(rt => + AddonRunConfigData( + rt.copy( + userId = insertData.runConfig.userId, + name = insertData.runConfig.name, + enabled = insertData.runConfig.enabled + ), + insertData.addons, + insertData.triggers + ) + ) + .semiflatMap(rt => + store.transact(AddonRunConfigData.update(rt).as(rt.runConfig.id)) + ) + .getOrElseF(store.transact(AddonRunConfigData.insert(insertData))) + } yield id + + EitherT(AddonRunConfigValidate(store, collective)(runConfig)) + .semiflatMap(_ => + upsert.flatTap { runConfigId => + runConfig.schedule match { + case Some(timer) => + userTasks.updateTask( + UserTaskScope.collective(collective), + s"Addon task ${runConfig.name}".some, + UserTask( + runConfigId, + scheduledAddonTaskName, + true, + timer, + s"Running scheduled addon task ${runConfig.name}".some, + ScheduledAddonTaskArgs(collective, runConfigId) + ) + ) + case None => + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + } + } + ) + .value + } + + def deleteAddonRunConfig(collective: Ident, runConfigId: Ident): F[Boolean] = { + val deleteRunConfig = + (for { + e <- OptionT(RAddonRunConfig.findById(collective, runConfigId)) + _ <- OptionT.liftF(RAddonRunConfigAddon.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfigTrigger.deleteAllForConfig(e.id)) + _ <- OptionT.liftF(RAddonRunConfig.deleteById(collective, e.id)) + } yield true).getOrElse(false) + + for { + deleted <- store.transact(deleteRunConfig) + _ <- + if (deleted) + userTasks.deleteTask(UserTaskScope.collective(collective), runConfigId) + else 0.pure[F] + } yield deleted + } + + def getAllAddons(collective: Ident): F[List[RAddonArchive]] = + store.transact(RAddonArchive.listAll(collective)) + + def deleteAddon(collective: Ident, addonId: Ident): F[Boolean] = + store.transact(RAddonArchive.deleteById(collective, addonId)).map(_ > 0) + + def inspectAddon( + collective: Ident, + url: LenientUri + ): F[AddonValidationResult[AddonMeta]] = + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + + def registerAddon( + collective: Ident, + url: LenientUri, + logger: Option[Logger[F]] + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val log = logger.getOrElse(this.logger) + def validateAndInsert(file: FileKey, localUrl: LenientUri) = + addonValidate.fromUrl(collective, url, urlReader, localUrl.some).flatMap { + case Right(meta) => + insertAddon(collective, url, meta, file) + .map(ar => AddonValidationResult.success(ar -> meta)) + + case Left(error) => + store.fileRepo + .delete(file) + .as(AddonValidationResult.failure[(RAddonArchive, AddonMeta)](error)) + } + + log.info(s"Store addon file from '${url.asString} for ${collective.id}") *> + storeAddonFromUrl(collective, url).flatMapF { file => + val localUrl = FileUrlReader.url(file) + for { + _ <- log.info(s"Validating addon…") + res <- validateAndInsert(file, localUrl) + _ <- log.info(s"Validation result: $res") + } yield res + }.value + } + + def refreshAddon( + collective: Ident, + addonId: Ident + ): F[AddonValidationResult[(RAddonArchive, AddonMeta)]] = { + val findAddon = store + .transact(RAddonArchive.findById(collective, addonId)) + .map(_.toRight(AddonNotFound)) + def validateAddon(aa: RAddonArchive): F[AddonValidationResult[AddonMeta]] = + aa.originalUrl.fold( + AddonValidationResult.failure[AddonMeta](RefreshLocalAddon).pure[F] + )(url => + addonValidate.fromUrl(collective, url, urlReader, checkExisting = false) + ) + + EitherT(findAddon).flatMap { aa => + EitherT(validateAddon(aa)) + .flatMap(meta => refreshAddon(aa, meta).map(na => na -> meta)) + }.value + } + + private def refreshAddon( + r: RAddonArchive, + meta: AddonMeta + ): EitherT[F, AddonValidationError, RAddonArchive] = + if (r.isUnchanged(meta)) EitherT.pure(r) + else + r.originalUrl match { + case Some(url) => + EitherT( + store + .transact( + RAddonArchive + .findByNameAndVersion(r.cid, meta.meta.name, meta.meta.version) + ) + .map( + _.fold(().asRight[AddonValidationError])(rx => AddonExists(rx).toLeft) + ) + ).flatMap(_ => + storeAddonFromUrl(r.cid, url).flatMap { file => + val nr = r.update(file, meta) + for { + _ <- EitherT( + store + .transact(RAddonArchive.update(nr)) + .map(_.asRight[AddonValidationError]) + .recoverWith { case ex => + logger.warn(ex)(s"Storing addon metadata failed.") *> + store.fileRepo + .delete(file) + .as( + AddonExists( + s"The addon '${nr.name}/${nr.version}' could not be stored", + nr + ).toLeft + ) + } + ) + _ <- EitherT.liftF(store.fileRepo.delete(r.fileId)) + } yield nr + } + ) + case None => + EitherT.leftT(RefreshLocalAddon.cast) + } + + private def insertAddon( + collective: Ident, + url: LenientUri, + meta: AddonMeta, + file: FileKey + ): F[RAddonArchive] = + for { + now <- Timestamp.current[F] + aId <- Ident.randomId[F] + record = RAddonArchive( + aId, + collective, + file, + url.some, + meta, + now + ) + _ <- store + .transact(RAddonArchive.insert(record, silent = false)) + .onError(_ => store.fileRepo.delete(file)) + } yield record + + private def storeAddonFromUrl(collective: Ident, url: LenientUri) = + for { + urlFile <- EitherT.pure(url.path.segments.lastOption) + file <- EitherT( + urlReader(url) + .through( + store.fileRepo.save( + collective, + FileCategory.Addon, + MimeTypeHint(urlFile, zip.some) + ) + ) + .compile + .lastOrError + .attempt + .map(_.leftMap(DownloadFailed(_).cast)) + ) + } yield file + + def runAddonForItem( + account: AccountId, + itemIds: NonEmptyList[Ident], + addonRunConfigIds: Set[Ident] + ): F[Unit] = + for { + jobs <- itemIds.traverse(id => + JobFactory.existingItemAddon( + ItemAddonTaskArgs(account.collective, id, addonRunConfigIds), + account + ) + ) + _ <- jobStore.insertAllIfNew(jobs.map(_.encode).toList) + } yield () + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala new file mode 100644 index 00000000..b5f0ddd8 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OAttachment.scala @@ -0,0 +1,223 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.data.{NonEmptyList => Nel, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.Stream + +import docspell.backend.JobFactory +import docspell.common.MakePreviewArgs.StoreMode +import docspell.common._ +import docspell.files.TikaMimetype +import docspell.ftsclient.{FtsClient, TextData} +import docspell.scheduler.JobStore +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +trait OAttachment[F[_]] { + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] +} + +object OAttachment { + + def apply[F[_]: Sync]( + store: Store[F], + fts: FtsClient[F], + jobStore: JobStore[F] + ): OAttachment[F] = + new OAttachment[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def setExtractedText( + collective: Ident, + itemId: Ident, + attachId: Ident, + newText: F[String] + ): F[Unit] = + for { + _ <- logger.info(s"Find attachment ${attachId.id} to update extracted text.") + cca <- store + .transact( + QAttachment + .allAttachmentMetaAndName( + collective.some, + Nel.of(itemId).some, + ItemState.validStates.append(ItemState.Processing), + 100 + ) + ) + .filter(_.id == attachId) + .compile + .last + content = cca.find(_.id == attachId) + _ <- logger.debug(s"Found existing metadata: ${content.isDefined}") + _ <- OptionT + .fromOption(content) + .semiflatMap { cnt => + for { + _ <- logger.debug(s"Setting new extracted text on ${cnt.id.id}") + text <- newText + td = TextData.attachment( + cnt.item, + cnt.id, + cnt.collective, + cnt.folder, + cnt.lang, + cnt.name, + text.some + ) + _ <- store.transact(RAttachmentMeta.updateContent(attachId, text)) + _ <- fts.updateIndex(logger, td) + } yield () + } + .getOrElseF( + logger.warn( + s"Item or attachment meta not found to update text: ${itemId.id}" + ) + ) + } yield () + + def addOrReplacePdf( + collective: Ident, + attachId: Ident, + pdfData: Stream[F, Byte], + regeneratePreview: Boolean + ): F[Unit] = { + def generatePreview(ra: RAttachment): F[Unit] = + JobFactory + .makePreview(MakePreviewArgs(ra.id, StoreMode.Replace), None) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to re-generate preview from new pdf") + + def generatePageCount(ra: RAttachment): F[Unit] = + JobFactory + .makePageCount( + MakePageCountArgs(ra.id), + AccountId(collective, DocspellSystem.user).some + ) + .map(_.encode) + .flatMap(jobStore.insert) *> + logger.info(s"Job submitted to find page count from new pdf") + + def setFile(ra: RAttachment, rs: RAttachmentSource) = + for { + _ <- requireMimeType(pdfData, MimeType.pdf) + + newFile <- pdfData + .through( + store.fileRepo.save( + collective, + FileCategory.AttachmentConvert, + MimeTypeHint.advertised(MimeType.pdf) + ) + ) + .compile + .lastOrError + + _ <- store.transact(RAttachment.updateFileId(attachId, newFile)) + _ <- logger.info(s"Deleting old file for attachment") + _ <- + if (rs.fileId == ra.fileId) ().pure[F] + else store.fileRepo.delete(ra.fileId) + _ <- + if (regeneratePreview) generatePreview(ra) + else ().pure[F] + _ <- generatePageCount(ra) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + rs <- OptionT( + store.transact(RAttachmentSource.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra, rs)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot replace pdf file. Attachment not found for id: ${attachId.id}" + ) + ) + } + + def addOrReplacePreview( + collective: Ident, + attachId: Ident, + imageData: Stream[F, Byte] + ): F[Unit] = { + def setFile(ra: RAttachment): F[Unit] = + for { + _ <- requireMimeType(imageData, MimeType.image("*")) + newFile <- imageData + .through( + store.fileRepo + .save(collective, FileCategory.PreviewImage, MimeTypeHint.none) + ) + .compile + .lastOrError + + now <- Timestamp.current[F] + record = RAttachmentPreview(ra.id, newFile, None, now) + oldFile <- store.transact(RAttachmentPreview.upsert(record)) + _ <- OptionT + .fromOption(oldFile) + .semiflatMap(store.fileRepo.delete) + .getOrElse(()) + } yield () + + (for { + ra <- OptionT( + store.transact(RAttachment.findByIdAndCollective(attachId, collective)) + ) + _ <- OptionT.liftF(setFile(ra)) + } yield ()).getOrElseF( + logger.warn( + s"Cannot add/replace preview file. Attachment not found for id: ${attachId.id}" + ) + ) + } + } + + private def requireMimeType[F[_]: Sync]( + data: Stream[F, Byte], + expectedMime: MimeType + ): F[Unit] = + TikaMimetype + .detect(data, MimeTypeHint.advertised(expectedMime)) + .flatMap { mime => + if (expectedMime.matches(mime)) ().pure[F] + else + Sync[F].raiseError( + new IllegalArgumentException( + s"Expected pdf file, but got: ${mime.asString}" + ) + ) + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 391a6593..79b799c0 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -61,6 +61,12 @@ trait OItem[F[_]] { collective: Ident ): F[AttachedEvent[UpdateResult]] + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -80,11 +86,13 @@ trait OItem[F[_]] { collective: Ident ): F[UpdateResult] - def setFolder(item: Ident, folder: Option[Ident], collective: Ident): F[UpdateResult] + /** Set or remove the folder on an item. Folder can be the id or name. */ + def setFolder(item: Ident, folder: Option[String], collective: Ident): F[UpdateResult] + /** Set or remove the folder on multiple items. Folder can be the id or name. */ def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] @@ -122,6 +130,13 @@ trait OItem[F[_]] { def setNotes(item: Ident, notes: Option[String], collective: Ident): F[UpdateResult] + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] def setNameMultiple( @@ -288,6 +303,28 @@ object OItem { } } + def removeTagsOfCategories( + item: Ident, + collective: Ident, + categories: Set[String] + ): F[AttachedEvent[UpdateResult]] = + if (categories.isEmpty) { + AttachedEvent.only(UpdateResult.success).pure[F] + } else { + val dbtask = + for { + tags <- RTag.findByItem(item) + removeTags = tags.filter(_.category.exists(categories.contains)) + _ <- RTagItem.removeAllTags(item, removeTags.map(_.tagId)) + mkEvent = Event.TagsChanged + .partial(Nel.of(item), Nil, removeTags.map(_.tagId.id).toList) + } yield AttachedEvent(UpdateResult.success)(mkEvent) + + OptionT(store.transact(RItem.checkByIdAndCollective(item, collective))) + .semiflatMap(_ => store.transact(dbtask)) + .getOrElse(AttachedEvent.only(UpdateResult.notFound)) + } + def removeTagsMultipleItems( items: Nel[Ident], tags: List[String], @@ -420,21 +457,27 @@ object OItem { def setFolder( item: Ident, - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = - UpdateResult - .fromUpdate( - store - .transact(RItem.updateFolder(item, collective, folder)) + for { + result <- store.transact(RItem.updateFolder(item, collective, folder)).attempt + ures = result.fold( + UpdateResult.failure, + t => UpdateResult.fromUpdateRows(t._1) ) - .flatTap( - onSuccessIgnoreError(fts.updateFolder(logger, item, collective, folder)) + _ <- result.fold( + _ => ().pure[F], + t => + onSuccessIgnoreError(fts.updateFolder(logger, item, collective, t._2))( + ures + ) ) + } yield ures def setFolderMultiple( items: Nel[Ident], - folder: Option[Ident], + folder: Option[String], collective: Ident ): F[UpdateResult] = for { @@ -615,6 +658,33 @@ object OItem { } ) + def addNotes( + item: Ident, + notes: String, + separator: Option[String], + collective: Ident + ): F[UpdateResult] = + store + .transact(RItem.appendNotes(item, collective, notes, separator)) + .flatMap { + case Some(newNotes) => + store + .transact(RCollective.findLanguage(collective)) + .map(_.getOrElse(Language.English)) + .flatMap(lang => + fts.updateItemNotes(logger, item, collective, lang, newNotes.some) + ) + .attempt + .flatMap { + case Right(()) => ().pure[F] + case Left(ex) => + logger.warn(s"Error updating full-text index: ${ex.getMessage}") + } + .as(UpdateResult.success) + case None => + UpdateResult.notFound.pure[F] + } + def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] = UpdateResult .fromUpdate( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index 9f83d46c..bc0b480c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -6,11 +6,13 @@ package docspell.backend.ops -import cats.Applicative import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import fs2.Stream -import docspell.common.Ident +import docspell.common.{Ident, NodeType} +import docspell.joexapi.client.JoexClient +import docspell.joexapi.model.AddonSupport import docspell.pubsub.api.PubSubT import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify} @@ -21,10 +23,16 @@ trait OJoex[F[_]] { def notifyPeriodicTasks: F[Unit] def cancelJob(job: Ident, worker: Ident): F[Unit] + + def getAddonSupport: F[List[AddonSupport]] } object OJoex { - def apply[F[_]: Applicative](pubSub: PubSubT[F]): Resource[F, OJoex[F]] = + def apply[F[_]: Async]( + pubSub: PubSubT[F], + nodes: ONode[F], + joexClient: JoexClient[F] + ): Resource[F, OJoex[F]] = Resource.pure[F, OJoex[F]](new OJoex[F] { def notifyAllNodes: F[Unit] = @@ -35,5 +43,17 @@ object OJoex { def cancelJob(job: Ident, worker: Ident): F[Unit] = pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(()) + + def getAddonSupport: F[List[AddonSupport]] = + for { + joex <- nodes.getNodes(NodeType.Joex) + conc = math.max(2, Runtime.getRuntime.availableProcessors() - 1) + supp <- Stream + .emits(joex) + .covary[F] + .parEvalMap(conc)(n => joexClient.getAddonSupport(n.url)) + .compile + .toList + } yield supp }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala index 8b55ed29..2e729c05 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/ONode.scala @@ -13,11 +13,27 @@ import docspell.common.{Ident, LenientUri, NodeType} import docspell.store.Store import docspell.store.records.RNode +import scodec.bits.ByteVector + trait ONode[F[_]] { - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] def unregister(appId: Ident): F[Unit] + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] + + def getNodes(nodeType: NodeType): F[Vector[RNode]] } object ONode { @@ -25,9 +41,14 @@ object ONode { def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] = Resource.pure[F, ONode[F]](new ONode[F] { val logger = docspell.logging.getLogger[F] - def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = + def register( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[Unit] = for { - node <- RNode(appId, nodeType, uri) + node <- RNode(appId, nodeType, uri, serverSecret) _ <- logger.info(s"Registering node ${node.id.id}") _ <- store.transact(RNode.set(node)) } yield () @@ -35,6 +56,19 @@ object ONode { def unregister(appId: Ident): F[Unit] = logger.info(s"Unregister app ${appId.id}") *> store.transact(RNode.delete(appId)).map(_ => ()) + + def withRegistered( + appId: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): Resource[F, Unit] = + Resource.make(register(appId, nodeType, uri, serverSecret))(_ => + unregister(appId) + ) + + def getNodes(nodeType: NodeType): F[Vector[RNode]] = + store.transact(RNode.findAll(nodeType)) }) } diff --git a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala index 00453680..fdf77e2b 100644 --- a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala +++ b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala @@ -9,8 +9,9 @@ package docspell.common import java.time.Instant import io.circe._ +import scodec.bits.ByteVector -object BaseJsonCodecs { +trait BaseJsonCodecs { implicit val encodeInstantEpoch: Encoder[Instant] = Encoder.encodeJavaLong.contramap(_.toEpochMilli) @@ -18,4 +19,11 @@ object BaseJsonCodecs { implicit val decodeInstantEpoch: Decoder[Instant] = Decoder.decodeLong.map(Instant.ofEpochMilli) + implicit val byteVectorEncoder: Encoder[ByteVector] = + Encoder.encodeString.contramap(_.toBase64) + + implicit val byteVectorDecoder: Decoder[ByteVector] = + Decoder.decodeString.emap(ByteVector.fromBase64Descriptive(_)) } + +object BaseJsonCodecs extends BaseJsonCodecs diff --git a/modules/common/src/main/scala/docspell/common/Binary.scala b/modules/common/src/main/scala/docspell/common/Binary.scala index 2dc3c354..fa36ed1c 100644 --- a/modules/common/src/main/scala/docspell/common/Binary.scala +++ b/modules/common/src/main/scala/docspell/common/Binary.scala @@ -18,6 +18,18 @@ final case class Binary[F[_]](name: String, mime: MimeType, data: Stream[F, Byte def withMime(mime: MimeType): Binary[F] = copy(mime = mime) + + /** Return the extension of `name` if available (without the dot) */ + def extension: Option[String] = + name.lastIndexOf('.') match { + case n if n > 0 => + Some(name.substring(n + 1)) + case _ => + None + } + + def extensionIn(extensions: Set[String]): Boolean = + extension.exists(extensions.contains) } object Binary { diff --git a/modules/common/src/main/scala/docspell/common/FileCategory.scala b/modules/common/src/main/scala/docspell/common/FileCategory.scala index cdf6c723..21caa25c 100644 --- a/modules/common/src/main/scala/docspell/common/FileCategory.scala +++ b/modules/common/src/main/scala/docspell/common/FileCategory.scala @@ -32,6 +32,7 @@ object FileCategory { case object PreviewImage extends FileCategory case object Classifier extends FileCategory case object DownloadAll extends FileCategory + case object Addon extends FileCategory val all: NonEmptyList[FileCategory] = NonEmptyList.of( @@ -39,7 +40,8 @@ object FileCategory { AttachmentConvert, PreviewImage, Classifier, - DownloadAll + DownloadAll, + Addon ) def fromString(str: String): Either[String, FileCategory] = diff --git a/modules/common/src/main/scala/docspell/common/Glob.scala b/modules/common/src/main/scala/docspell/common/Glob.scala index 722af9fd..5b7ee9ca 100644 --- a/modules/common/src/main/scala/docspell/common/Glob.scala +++ b/modules/common/src/main/scala/docspell/common/Glob.scala @@ -32,7 +32,8 @@ object Glob { def single(str: String) = PatternGlob(Pattern(split(str, separator).map(makeSegment))) - if (in == "*") all + if (in == all.asString) all + else if (in == none.asString) none else split(in, anyChar) match { case NonEmptyList(_, Nil) => @@ -51,15 +52,25 @@ object Glob { val asString = "*" } + val none = new Glob { + def matches(caseSensitive: Boolean)(in: String) = false + def matchFilenameOrPath(in: String) = false + def asString = "!*" + } + def pattern(pattern: Pattern): Glob = PatternGlob(pattern) /** A simple glob supporting `*` and `?`. */ final private case class PatternGlob(pattern: Pattern) extends Glob { - def matches(caseSensitive: Boolean)(in: String): Boolean = + def matches(caseSensitive: Boolean)(in: String): Boolean = { + val input = Glob.split(in, Glob.separator) + + pattern.parts.size == input.size && pattern.parts - .zipWith(Glob.split(in, Glob.separator))(_.matches(caseSensitive)(_)) + .zipWith(input)(_.matches(caseSensitive)(_)) .forall(identity) + } def matchFilenameOrPath(in: String): Boolean = if (pattern.parts.tail.isEmpty) matches(true)(split(in, separator).last) @@ -67,6 +78,8 @@ object Glob { def asString: String = pattern.asString + + override def toString = s"PatternGlob($asString)" } final private case class AnyGlob(globs: NonEmptyList[Glob]) extends Glob { @@ -76,6 +89,8 @@ object Glob { globs.exists(_.matchFilenameOrPath(in)) def asString = globs.toList.map(_.asString).mkString(anyChar.toString) + + override def toString = s"AnyGlob($globs)" } case class Pattern(parts: NonEmptyList[Segment]) { diff --git a/modules/common/src/main/scala/docspell/common/Ident.scala b/modules/common/src/main/scala/docspell/common/Ident.scala index 2f630e33..928f0dd5 100644 --- a/modules/common/src/main/scala/docspell/common/Ident.scala +++ b/modules/common/src/main/scala/docspell/common/Ident.scala @@ -26,6 +26,9 @@ case class Ident(id: String) { def /(next: Ident): Ident = new Ident(id + Ident.concatChar + next.id) + + def take(n: Int): Ident = + new Ident(id.take(n)) } object Ident { diff --git a/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala new file mode 100644 index 00000000..3333486d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ItemAddonTaskArgs.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +/** Arguments to submit a task that runs addons configured for some existing item. + * + * If `addonTaskIds` is non empty, only these addon tasks are run. Otherwise all addon + * tasks that are configured for 'existing-item' are run. + */ +final case class ItemAddonTaskArgs( + collective: Ident, + itemId: Ident, + addonRunConfigs: Set[Ident] +) + +object ItemAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-existing-item") + + implicit val jsonDecoder: Decoder[ItemAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala index 55fbbda1..72da5281 100644 --- a/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala +++ b/modules/common/src/main/scala/docspell/common/MimeTypeHint.scala @@ -6,6 +6,8 @@ package docspell.common +import fs2.io.file.Path + case class MimeTypeHint(filename: Option[String], advertised: Option[String]) { def withName(name: String): MimeTypeHint = @@ -21,6 +23,9 @@ object MimeTypeHint { def filename(name: String): MimeTypeHint = MimeTypeHint(Some(name), None) + def filename(file: Path): MimeTypeHint = + filename(file.fileName.toString) + def advertised(mimeType: MimeType): MimeTypeHint = advertised(mimeType.asString) diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index 8398eaf0..9a92cee7 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -17,7 +17,7 @@ import io.circe.generic.semiauto._ * This task is run for each new file to create a new item from it or to add this file as * an attachment to an existing item. * - * If the `itemId' is set to some value, the item is tried to load to ammend with the + * If the `itemId' is set to some value, the item is tried to load to amend with the * given files. Otherwise a new item is created. * * It is also re-used by the 'ReProcessItem' task. diff --git a/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala new file mode 100644 index 00000000..5104108e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ScheduledAddonTaskArgs.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class ScheduledAddonTaskArgs(collective: Ident, addonTaskId: Ident) + +object ScheduledAddonTaskArgs { + val taskName: Ident = Ident.unsafe("addon-scheduled-task") + + implicit val jsonDecoder: Decoder[ScheduledAddonTaskArgs] = deriveDecoder + implicit val jsonEncoder: Encoder[ScheduledAddonTaskArgs] = deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/SystemCommand.scala b/modules/common/src/main/scala/docspell/common/SystemCommand.scala index 63f4ca40..702c546d 100644 --- a/modules/common/src/main/scala/docspell/common/SystemCommand.scala +++ b/modules/common/src/main/scala/docspell/common/SystemCommand.scala @@ -17,11 +17,23 @@ import cats.implicits._ import fs2.io.file.Path import fs2.{Stream, io, text} +import docspell.common.{exec => newExec} import docspell.logging.Logger +// better use `SysCmd` and `SysExec` object SystemCommand { - final case class Config(program: String, args: Seq[String], timeout: Duration) { + final case class Config( + program: String, + args: Seq[String], + timeout: Duration, + env: Map[String, String] = Map.empty + ) { + + def toSysCmd = newExec + .SysCmd(program, newExec.Args(args)) + .withTimeout(timeout) + .addEnv(newExec.Env(env)) def mapArgs(f: String => String): Config = Config(program, args.map(f), timeout) @@ -33,6 +45,18 @@ object SystemCommand { } ) + def withEnv(key: String, value: String): Config = + copy(env = env.updated(key, value)) + + def addEnv(moreEnv: Map[String, String]): Config = + copy(env = env ++ moreEnv) + + def appendArgs(extraArgs: Args): Config = + copy(args = args ++ extraArgs.args) + + def appendArgs(extraArgs: Seq[String]): Config = + copy(args = args ++ extraArgs) + def toCmd: List[String] = program :: args.toList @@ -40,6 +64,45 @@ object SystemCommand { toCmd.mkString(" ") } + final case class Args(args: Vector[String]) extends Iterable[String] { + override def iterator = args.iterator + + def prepend(a: String): Args = Args(a +: args) + + def prependWhen(flag: Boolean)(a: String): Args = + prependOption(Option.when(flag)(a)) + + def prependOption(value: Option[String]): Args = + value.map(prepend).getOrElse(this) + + def append(a: String, as: String*): Args = + Args(args ++ (a +: as.toVector)) + + def appendOption(value: Option[String]): Args = + value.map(append(_)).getOrElse(this) + + def appendOptionVal(first: String, second: Option[String]): Args = + second.map(b => append(first, b)).getOrElse(this) + + def appendWhen(flag: Boolean)(a: String, as: String*): Args = + if (flag) append(a, as: _*) else this + + def appendWhenNot(flag: Boolean)(a: String, as: String*): Args = + if (!flag) append(a, as: _*) else this + + def append(p: Path): Args = + append(p.toString) + + def append(as: Iterable[String]): Args = + Args(args ++ as.toVector) + } + object Args { + val empty: Args = Args() + + def apply(as: String*): Args = + Args(as.toVector) + } + final case class Result(rc: Int, stdout: String, stderr: String) def exec[F[_]: Sync]( @@ -104,6 +167,10 @@ object SystemCommand { .redirectError(Redirect.PIPE) .redirectOutput(Redirect.PIPE) + val pbEnv = pb.environment() + cmd.env.foreach { case (key, value) => + pbEnv.put(key, value) + } wd.map(_.toNioPath.toFile).foreach(pb.directory) pb.start() } diff --git a/modules/common/src/main/scala/docspell/common/UrlMatcher.scala b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala new file mode 100644 index 00000000..c8fd393e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlMatcher.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList +import cats.kernel.Monoid +import cats.syntax.all._ + +trait UrlMatcher { + def matches(url: LenientUri): Boolean +} + +object UrlMatcher { + val True = instance(_ => true) + val False = instance(_ => false) + + def instance(f: LenientUri => Boolean): UrlMatcher = + (url: LenientUri) => f(url) + + def fromString(str: String): Either[String, UrlMatcher] = + if (str == "") False.asRight + else if (str == "*") True.asRight + else LenientUri.parse(str).map(fromUrl) + + def unsafeFromString(str: String): UrlMatcher = + fromString(str).fold(sys.error, identity) + + def fromStringList(str: List[String]): Either[String, UrlMatcher] = + str match { + case Nil => False.asRight + case _ => str.map(_.trim).traverse(fromString).map(_.combineAll) + } + + def fromUrl(url: LenientUri): UrlMatcher = { + val schemeGlob = Glob(url.scheme.head) + val hostGlob = HostGlob(url.host) + val pathGlob = Glob(url.path.asString) + new Impl(schemeGlob, hostGlob, pathGlob, url.path.segments.size) + } + + def any(ulrm: IterableOnce[UrlMatcher]): UrlMatcher = + anyMonoid.combineAll(ulrm) + + def all(urlm: IterableOnce[UrlMatcher]): UrlMatcher = + allMonoid.combineAll(urlm) + + val anyMonoid: Monoid[UrlMatcher] = + Monoid.instance(False, (a, b) => instance(url => a.matches(url) || b.matches(url))) + + val allMonoid: Monoid[UrlMatcher] = + Monoid.instance(True, (a, b) => instance(url => a.matches(url) && b.matches(url))) + + implicit val defaultMonoid: Monoid[UrlMatcher] = anyMonoid + + private class Impl(scheme: Glob, host: HostGlob, path: Glob, pathSegmentCount: Int) + extends UrlMatcher { + def matches(url: LenientUri) = { + // strip path to only match prefixes + val mPath: LenientUri.Path = + NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match { + case Some(nel) => LenientUri.NonEmptyPath(nel) + case None => LenientUri.RootPath + } + + url.scheme.forall(scheme.matches(false)) && + host.matches(url.host) && + path.matchFilenameOrPath(mPath.asString) + } + } + + private class HostGlob(glob: Option[Glob]) { + def matches(host: Option[String]): Boolean = + (glob, host) match { + case (Some(pattern), Some(word)) => + pattern.matches(false)(HostGlob.prepareHost(word)) + case (None, None) => true + case _ => false + } + + override def toString = s"HostGlob(${glob.map(_.asString)})" + } + + private object HostGlob { + def apply(hostPattern: Option[String]): HostGlob = + new HostGlob(hostPattern.map(p => Glob(prepareHost(p)))) + + private def prepareHost(host: String): String = + host.replace('.', '/') + } +} diff --git a/modules/common/src/main/scala/docspell/common/UrlReader.scala b/modules/common/src/main/scala/docspell/common/UrlReader.scala new file mode 100644 index 00000000..7e1521f2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UrlReader.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.ApplicativeError +import cats.effect._ +import fs2.Stream + +trait UrlReader[F[_]] { + def apply(url: LenientUri): Stream[F, Byte] +} + +object UrlReader { + + def instance[F[_]](f: LenientUri => Stream[F, Byte]): UrlReader[F] = + (url: LenientUri) => f(url) + + def failWith[F[_]]( + message: String + )(implicit F: ApplicativeError[F, Throwable]): UrlReader[F] = + instance(url => + Stream.raiseError( + new IllegalStateException(s"Unable to read '${url.asString}': $message") + ) + ) + + def apply[F[_]](implicit r: UrlReader[F]): UrlReader[F] = r + + implicit def defaultReader[F[_]: Sync]: UrlReader[F] = + instance(_.readURL[F](8192)) +} diff --git a/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala new file mode 100644 index 00000000..f0caa3a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/AttachmentAction.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait AttachmentAction {} + +object AttachmentAction { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class SetExtractedText(text: Option[String]) extends AttachmentAction + object SetExtractedText { + implicit val jsonDecoder: Decoder[SetExtractedText] = deriveDecoder + implicit val jsonEncoder: Encoder[SetExtractedText] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[AttachmentAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[AttachmentAction] = deriveConfiguredEncoder + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala new file mode 100644 index 00000000..f5732f64 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommand.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait BackendCommand {} + +object BackendCommand { + + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("command").withKebabCaseConstructorNames + + case class ItemUpdate(itemId: Ident, actions: List[ItemAction]) extends BackendCommand + object ItemUpdate { + implicit val jsonDecoder: Decoder[ItemUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[ItemUpdate] = deriveEncoder + } + + def item(itemId: Ident, actions: List[ItemAction]): BackendCommand = + ItemUpdate(itemId, actions) + + case class AttachmentUpdate( + itemId: Ident, + attachId: Ident, + actions: List[AttachmentAction] + ) extends BackendCommand + object AttachmentUpdate { + implicit val jsonDecoder: Decoder[AttachmentUpdate] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachmentUpdate] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[BackendCommand] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[BackendCommand] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala new file mode 100644 index 00000000..863e35ed --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/BackendCommandRunner.scala @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +trait BackendCommandRunner[F[_], A] { + + def run(collective: Ident, cmd: BackendCommand): F[A] + + def runAll(collective: Ident, cmds: List[BackendCommand]): F[A] + +} diff --git a/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala new file mode 100644 index 00000000..16a09048 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/bc/ItemAction.scala @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common.Ident + +import io.circe.generic.extras.Configuration +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +sealed trait ItemAction {} + +object ItemAction { + implicit val deriveConfig: Configuration = + Configuration.default.withDiscriminator("action").withKebabCaseConstructorNames + + case class AddTags(tags: Set[String]) extends ItemAction + object AddTags { + implicit val jsonDecoder: Decoder[AddTags] = deriveDecoder + implicit val jsonEncoder: Encoder[AddTags] = deriveEncoder + } + + case class ReplaceTags(tags: Set[String]) extends ItemAction + object ReplaceTags { + implicit val jsonDecoder: Decoder[ReplaceTags] = deriveDecoder + implicit val jsonEncoder: Encoder[ReplaceTags] = deriveEncoder + } + + case class RemoveTags(tags: Set[String]) extends ItemAction + object RemoveTags { + implicit val jsonDecoder: Decoder[RemoveTags] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTags] = deriveEncoder + } + + case class RemoveTagsCategory(categories: Set[String]) extends ItemAction + object RemoveTagsCategory { + implicit val jsonDecoder: Decoder[RemoveTagsCategory] = deriveDecoder + implicit val jsonEncoder: Encoder[RemoveTagsCategory] = deriveEncoder + } + + case class SetFolder(folder: Option[String]) extends ItemAction + object SetFolder { + implicit val jsonDecoder: Decoder[SetFolder] = deriveDecoder + implicit val jsonEncoder: Encoder[SetFolder] = deriveEncoder + } + + case class SetCorrOrg(id: Option[Ident]) extends ItemAction + object SetCorrOrg { + implicit val jsonDecoder: Decoder[SetCorrOrg] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrOrg] = deriveEncoder + } + + case class SetCorrPerson(id: Option[Ident]) extends ItemAction + object SetCorrPerson { + implicit val jsonDecoder: Decoder[SetCorrPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetCorrPerson] = deriveEncoder + } + + case class SetConcPerson(id: Option[Ident]) extends ItemAction + object SetConcPerson { + implicit val jsonDecoder: Decoder[SetConcPerson] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcPerson] = deriveEncoder + } + + case class SetConcEquipment(id: Option[Ident]) extends ItemAction + object SetConcEquipment { + implicit val jsonDecoder: Decoder[SetConcEquipment] = deriveDecoder + implicit val jsonEncoder: Encoder[SetConcEquipment] = deriveEncoder + } + + case class SetField(field: Ident, value: String) extends ItemAction + object SetField { + implicit val jsonDecoder: Decoder[SetField] = deriveDecoder + implicit val jsonEncoder: Encoder[SetField] = deriveEncoder + } + + case class SetName(name: String) extends ItemAction + object SetName { + implicit val jsonDecoder: Decoder[SetName] = deriveDecoder + implicit val jsonEncoder: Encoder[SetName] = deriveEncoder + } + + case class SetNotes(notes: Option[String]) extends ItemAction + object SetNotes { + implicit val jsonDecoder: Decoder[SetNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[SetNotes] = deriveEncoder + } + + case class AddNotes(notes: String, separator: Option[String]) extends ItemAction + object AddNotes { + implicit val jsonDecoder: Decoder[AddNotes] = deriveDecoder + implicit val jsonEncoder: Encoder[AddNotes] = deriveEncoder + } + + implicit val jsonDecoder: Decoder[ItemAction] = deriveConfiguredDecoder + implicit val jsonEncoder: Encoder[ItemAction] = deriveConfiguredEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Args.scala b/modules/common/src/main/scala/docspell/common/exec/Args.scala new file mode 100644 index 00000000..292899a1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Args.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import fs2.io.file.Path + +case class Args(values: Seq[String]) { + + def option(key: String, value: String): Args = + Args(values ++ Seq(key, value)) + + def option(key: String, value: Option[String]): Args = + value.map(v => option(key, v)).getOrElse(this) + + def appendOpt(v: Option[String]): Args = + v.map(e => Args(values :+ e)).getOrElse(this) + + def append(v: String, vs: String*): Args = + Args(values ++ (v +: vs)) + + def append(path: Path): Args = + append(path.toString) + + def append(args: Args): Args = + Args(values ++ args.values) + + def append(args: Seq[String]): Args = + Args(values ++ args) + + def prepend(v: String): Args = + Args(v +: values) + + def prependWhen(flag: Boolean)(v: String) = + if (flag) prepend(v) else this + + def cmdString: String = + values.mkString(" ") +} + +object Args { + val empty: Args = Args(Seq.empty) + + def of(v: String*): Args = + Args(v) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/Env.scala b/modules/common/src/main/scala/docspell/common/exec/Env.scala new file mode 100644 index 00000000..2524d35a --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/Env.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +case class Env(values: Map[String, String]) { + + def add(name: String, value: String): Env = + copy(values.updated(name, value)) + + def addAll(v: Map[String, String]): Env = + Env(values ++ v) + + def addAll(e: Env): Env = + Env(values ++ e.values) + + def ++(e: Env) = addAll(e) + + def foreach(f: (String, String) => Unit): Unit = + values.foreach(t => f(t._1, t._2)) + + def map[A](f: (String, String) => A): Seq[A] = + values.map(f.tupled).toSeq + + def mapConcat[A](f: (String, String) => Seq[A]): Seq[A] = + values.flatMap(f.tupled).toSeq +} + +object Env { + val empty: Env = Env(Map.empty) + + def of(nv: (String, String)*): Env = + Env(Map(nv: _*)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala new file mode 100644 index 00000000..b94b8c4b --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysCmd.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import docspell.common._ + +final case class SysCmd( + program: String, + args: Args, + env: Env, + timeout: Duration +) { + + def withArgs(f: Args => Args): SysCmd = + copy(args = f(args)) + + def withTimeout(to: Duration): SysCmd = + copy(timeout = to) + + def withEnv(f: Env => Env): SysCmd = + copy(env = f(env)) + + def addEnv(env: Env): SysCmd = + withEnv(_.addAll(env)) + + def cmdString: String = + s"$program ${args.cmdString}" + + private[exec] def toCmd: Seq[String] = + program +: args.values +} + +object SysCmd { + def apply(prg: String, args: String*): SysCmd = + apply(prg, Args(args)) + + def apply(prg: String, args: Args): SysCmd = + SysCmd(prg, args, Env.empty, Duration.minutes(2)) +} diff --git a/modules/common/src/main/scala/docspell/common/exec/SysExec.scala b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala new file mode 100644 index 00000000..da7b10c2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/exec/SysExec.scala @@ -0,0 +1,163 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.exec + +import java.lang.ProcessBuilder.Redirect +import java.util.concurrent.TimeUnit + +import scala.concurrent.TimeoutException +import scala.jdk.CollectionConverters._ + +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Path +import fs2.{Pipe, Stream} + +import docspell.common.Duration +import docspell.logging.Logger + +trait SysExec[F[_]] { + + def stdout: Stream[F, Byte] + + def stdoutLines: Stream[F, String] = + stdout + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def stderr: Stream[F, Byte] + + def stderrLines: Stream[F, String] = + stderr + .through(fs2.text.utf8.decode) + .through(fs2.text.lines) + + def waitFor(timeout: Option[Duration] = None): F[Int] + + /** Sends a signal to the process to terminate it immediately */ + def cancel: F[Unit] + + /** Consume lines of output of the process in background. */ + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] + + /** Consumes stderr lines (left) and stdout lines (right) in a background thread. */ + def consumeOutputs( + m: Either[String, String] => F[Unit] + )(implicit F: Async[F]): Resource[F, SysExec[F]] = { + val pe: Pipe[F, String, Unit] = _.map(_.asLeft).evalMap(m) + val po: Pipe[F, String, Unit] = _.map(_.asRight).evalMap(m) + consumeOutputs(po, pe) + } + + def logOutputs(logger: Logger[F], name: String)(implicit F: Async[F]) = + consumeOutputs { + case Right(line) => logger.debug(s"[$name (out)]: $line") + case Left(line) => logger.debug(s"[$name (err)]: $line") + } +} + +object SysExec { + private val readChunkSz = 8 * 1024 + + def apply[F[_]: Sync]( + cmd: SysCmd, + logger: Logger[F], + workdir: Option[Path] = None, + stdin: Option[Stream[F, Byte]] = None + ): Resource[F, SysExec[F]] = + for { + proc <- startProcess(logger, cmd, workdir, stdin) + fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil)) + } yield new SysExec[F] { + def stdout: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getInputStream), + readChunkSz, + closeAfterUse = false + ) + + def stderr: Stream[F, Byte] = + fs2.io.readInputStream( + Sync[F].blocking(proc.getErrorStream), + readChunkSz, + closeAfterUse = false + ) + + def cancel = Sync[F].blocking(proc.destroy()) + + def waitFor(timeout: Option[Duration]): F[Int] = { + val to = timeout.getOrElse(cmd.timeout) + logger.trace("Waiting for command to terminate…") *> + Sync[F] + .blocking(proc.waitFor(to.millis, TimeUnit.MILLISECONDS)) + .flatTap(_ => fibers.get.flatMap(_.traverse_(identity))) + .flatMap(terminated => + if (terminated) proc.exitValue().pure[F] + else + Sync[F] + .raiseError( + new TimeoutException(s"Timed out after: ${to.formatExact}") + ) + ) + } + + def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit + F: Async[F] + ): Resource[F, SysExec[F]] = + for { + f1 <- F.background(stdoutLines.through(out).compile.drain) + f2 <- F.background(stderrLines.through(err).compile.drain) + _ <- Resource.eval(fibers.update(list => f1.void :: f2.void :: list)) + } yield this + } + + private def startProcess[F[_]: Sync, A]( + logger: Logger[F], + cmd: SysCmd, + workdir: Option[Path], + stdin: Option[Stream[F, Byte]] + ): Resource[F, Process] = { + val log = logger.debug(s"Running external command: ${cmd.cmdString}") + + val proc = log *> + Sync[F].blocking { + val pb = new ProcessBuilder(cmd.toCmd.asJava) + .redirectInput(if (stdin.isDefined) Redirect.PIPE else Redirect.INHERIT) + .redirectError(Redirect.PIPE) + .redirectOutput(Redirect.PIPE) + + val pbEnv = pb.environment() + cmd.env.foreach { (name, v) => + pbEnv.put(name, v) + () + } + workdir.map(_.toNioPath.toFile).foreach(pb.directory) + pb.start() + } + + Resource + .make(proc)(p => + logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy()) + ) + .evalMap(p => + stdin match { + case Some(in) => + writeToProcess(in, p).compile.drain.as(p) + case None => + p.pure[F] + } + ) + } + + private def writeToProcess[F[_]: Sync]( + data: Stream[F, Byte], + proc: Process + ): Stream[F, Nothing] = + data.through(fs2.io.writeOutputStream(Sync[F].blocking(proc.getOutputStream))) +} diff --git a/modules/common/src/main/scala/docspell/common/File.scala b/modules/common/src/main/scala/docspell/common/util/File.scala similarity index 91% rename from modules/common/src/main/scala/docspell/common/File.scala rename to modules/common/src/main/scala/docspell/common/util/File.scala index 0d6505dd..679c99ec 100644 --- a/modules/common/src/main/scala/docspell/common/File.scala +++ b/modules/common/src/main/scala/docspell/common/util/File.scala @@ -4,20 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.common +package docspell.common.util import java.nio.file.{Path => JPath} -import cats.FlatMap -import cats.Monad import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ +import cats.{FlatMap, Monad} import fs2.Stream import fs2.io.file.{Files, Flags, Path} -import docspell.common.syntax.all._ - import io.circe.Decoder +import io.circe.parser object File { @@ -75,6 +73,5 @@ object File { .map(_ => file) def readJson[F[_]: Async, A](file: Path)(implicit d: Decoder[A]): F[A] = - readText[F](file).map(_.parseJsonAs[A]).rethrow - + readText[F](file).map(parser.decode[A]).rethrow } diff --git a/modules/common/src/main/scala/docspell/common/util/Random.scala b/modules/common/src/main/scala/docspell/common/util/Random.scala new file mode 100644 index 00000000..b192aa6d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/util/Random.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.util + +import cats.effect._ + +import scodec.bits.ByteVector + +trait Random[F[_]] { + def string(len: Int): F[String] + def string: F[String] = string(8) +} + +object Random { + def apply[F[_]: Sync] = + new Random[F] { + def string(len: Int) = Sync[F].delay { + val buf = Array.ofDim[Byte](len) + new scala.util.Random().nextBytes(buf) + ByteVector.view(buf).toBase58 + } + } +} diff --git a/modules/common/src/test/scala/docspell/common/GlobTest.scala b/modules/common/src/test/scala/docspell/common/GlobTest.scala index 7e3d3bf8..4ee7d468 100644 --- a/modules/common/src/test/scala/docspell/common/GlobTest.scala +++ b/modules/common/src/test/scala/docspell/common/GlobTest.scala @@ -70,11 +70,13 @@ class GlobTest extends FunSuite { test("with splitting") { assert(Glob("a/b/*").matches(true)("a/b/hello")) + assert(!Glob("a/b/*").matches(true)("a/b/hello/bello")) assert(!Glob("a/b/*").matches(true)("/a/b/hello")) assert(Glob("/a/b/*").matches(true)("/a/b/hello")) assert(!Glob("/a/b/*").matches(true)("a/b/hello")) assert(!Glob("*/a/b/*").matches(true)("a/b/hello")) assert(Glob("*/a/b/*").matches(true)("test/a/b/hello")) + assert(!Glob("/a/b").matches(true)("/a/b/c/d")) } test("asString") { diff --git a/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala new file mode 100644 index 00000000..51fb1f19 --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/UrlMatcherTest.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import munit._ + +class UrlMatcherTest extends FunSuite { + + test("it should match patterns") { + assertUrlsMatch( + uri("https://github.com/docspell/*") -> uri("https://github.com/docspell/dsc"), + uri("*s://test.com/*") -> uri("https://test.com/a"), + uri("*s://test.com/*") -> uri("https://test.com/a/b"), + uri("*s://test.com/*") -> uri("https://test.com/a/b/c"), + uri("*s://test.com/project/*") -> uri("https://test.com/project/c"), + uri("https://*.test.com/projects/*") -> uri("https://a.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1"), + uri("https://*.test.com/projects/*") -> uri("https://b.test.com/projects/p1") + ) + + assertUrlsNotMatch( + uri("https://*.test.com/projects/*") -> uri("https://test.com/projects/p1"), + uri("*s://test.com/project/*") -> uri("https://test.com/subject/c") + ) + } + + def uri(str: String): LenientUri = LenientUri.unsafe(str) + + def assertUrlsMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri does not match $checkUri" + ) + } + + def assertUrlsMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsMatch(test :: more.toList) + + def assertUrlsNotMatch(tests: List[(LenientUri, LenientUri)]): Unit = + tests.foreach { case (patternUri, checkUri) => + assert( + !UrlMatcher.fromUrl(patternUri).matches(checkUri), + s"$patternUri incorrectly matches $checkUri" + ) + } + + def assertUrlsNotMatch( + test: (LenientUri, LenientUri), + more: (LenientUri, LenientUri)* + ): Unit = + assertUrlsNotMatch(test :: more.toList) +} diff --git a/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala new file mode 100644 index 00000000..bb0cfaef --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/bc/BackendCommandTest.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common.bc + +import docspell.common._ + +import io.circe.parser +import io.circe.syntax._ +import munit._ + +class BackendCommandTest extends FunSuite { + + test("encode json") { + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals( + bc.asJson.spaces2, + """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + ) + } + + test("decode case insensitive keys") { + val json = """{ + | "itemId" : "abc", + | "actions" : [ + | { + | "categories" : [ + | "doctype" + | ], + | "action" : "remove-tags-category" + | }, + | { + | "tags" : [ + | "tag1", + | "tag2" + | ], + | "action" : "add-tags" + | } + | ], + | "command" : "item-update" + |}""".stripMargin + + val bc: BackendCommand = + BackendCommand.item( + id("abc"), + List( + ItemAction.RemoveTagsCategory(Set("doctype")), + ItemAction.AddTags(Set("tag1", "tag2")) + ) + ) + + assertEquals(parser.decode[BackendCommand](json), Right(bc)) + } + + def id(str: String) = Ident.unsafe(str) +} diff --git a/modules/config/src/main/scala/docspell/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala index e529a2c5..014643ae 100644 --- a/modules/config/src/main/scala/docspell/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -13,6 +13,7 @@ import scala.reflect.ClassTag import cats.syntax.all._ import fs2.io.file.Path +import docspell.addons.RunnerType import docspell.common._ import docspell.ftspsql.{PgQueryParser, RankNormalization} import docspell.logging.{Level, LogConfig} @@ -32,6 +33,17 @@ object Implicits { else super.fieldValue(name) } + implicit val urlMatcherReader: ConfigReader[UrlMatcher] = { + val fromList = ConfigReader[List[String]].emap(reason(UrlMatcher.fromStringList)) + val fromString = ConfigReader[String].emap( + reason(str => UrlMatcher.fromStringList(str.split("[\\s,]+").toList)) + ) + fromList.orElse(fromString) + } + + implicit val runnerSelectReader: ConfigReader[List[RunnerType]] = + ConfigReader[String].emap(reason(RunnerType.fromSeparatedString)) + implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) diff --git a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala index eff5a0fb..8f0f9e11 100644 --- a/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala +++ b/modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala @@ -12,6 +12,7 @@ import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt} import docspell.logging.Logger diff --git a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala index 25905afe..06ee81c1 100644 --- a/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/ConversionTest.scala @@ -15,6 +15,7 @@ import cats.implicits._ import fs2.Stream import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.convert.extern.OcrMyPdfConfig import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig} diff --git a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala index 96f251ff..ad41c01c 100644 --- a/modules/convert/src/test/scala/docspell/convert/FileChecks.scala +++ b/modules/convert/src/test/scala/docspell/convert/FileChecks.scala @@ -18,6 +18,7 @@ import fs2.io.file.Path import fs2.{Pipe, Stream} import docspell.common._ +import docspell.common.util.File import docspell.convert.ConversionResult.Handler import docspell.files.TikaMimetype diff --git a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala index 6f0ab2ab..9beaed28 100644 --- a/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala +++ b/modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala @@ -14,6 +14,7 @@ import cats.effect.unsafe.implicits.global import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.convert._ import docspell.files.ExampleFiles import docspell.logging.TestLoggingConfig diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala index 727666a8..f39e4b0b 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala @@ -11,6 +11,7 @@ import fs2.Stream import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger object Ocr { diff --git a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala index 003fa8c3..856c21a3 100644 --- a/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala +++ b/modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala @@ -11,6 +11,7 @@ import java.nio.file.Paths import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File case class OcrConfig( maxImageSize: Int, diff --git a/modules/files/src/main/scala/docspell/files/FileSupport.scala b/modules/files/src/main/scala/docspell/files/FileSupport.scala new file mode 100644 index 00000000..5219ac37 --- /dev/null +++ b/modules/files/src/main/scala/docspell/files/FileSupport.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.files + +import cats.data.OptionT +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream +import fs2.io.file.{Files, Path} + +import docspell.common.{MimeType, MimeTypeHint} + +import io.circe.Encoder +import io.circe.syntax._ + +trait FileSupport { + implicit final class FileOps[F[_]: Files: Sync](self: Path) { + def detectMime: F[Option[MimeType]] = + Files[F].isReadable(self).flatMap { flag => + OptionT + .whenF(flag) { + TikaMimetype + .detect( + Files[F].readAll(self), + MimeTypeHint.filename(self.fileName.toString) + ) + } + .value + } + + def asTextFile(alt: MimeType => F[Unit]): F[Option[Path]] = + OptionT(detectMime).flatMapF { mime => + if (mime.matches(MimeType.text("plain"))) self.some.pure[F] + else alt(mime).as(None: Option[Path]) + }.value + + def readText: F[String] = + Files[F] + .readAll(self) + .through(fs2.text.utf8.decode) + .compile + .string + + def readAll: Stream[F, Byte] = + Files[F].readAll(self) + + def writeJson[A: Encoder](value: A): F[Unit] = + Stream + .emit(value.asJson.noSpaces) + .through(fs2.text.utf8.encode) + .through(Files[F].writeAll(self)) + .compile + .drain + } +} + +object FileSupport extends FileSupport diff --git a/modules/files/src/main/scala/docspell/files/Zip.scala b/modules/files/src/main/scala/docspell/files/Zip.scala index 3fd938e4..b8d1dfd0 100644 --- a/modules/files/src/main/scala/docspell/files/Zip.scala +++ b/modules/files/src/main/scala/docspell/files/Zip.scala @@ -8,11 +8,12 @@ package docspell.files import java.io.InputStream import java.nio.charset.StandardCharsets -import java.nio.file.Paths import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} +import cats.data.OptionT import cats.effect._ import cats.implicits._ +import fs2.io.file.{Files, Path} import fs2.{Pipe, Stream} import docspell.common.Binary @@ -27,16 +28,72 @@ object Zip { ): Pipe[F, (String, Stream[F, Byte]), Byte] = in => zipJava(logger, chunkSize, in.through(deduplicate)) - def unzipP[F[_]: Async](chunkSize: Int, glob: Glob): Pipe[F, Byte, Binary[F]] = - s => unzip[F](chunkSize, glob)(s) + def unzip[F[_]: Async]( + chunkSize: Int, + glob: Glob + ): Pipe[F, Byte, Binary[F]] = + s => unzipStream[F](chunkSize, glob)(s) - def unzip[F[_]: Async](chunkSize: Int, glob: Glob)( + def unzipStream[F[_]: Async](chunkSize: Int, glob: Glob)( data: Stream[F, Byte] ): Stream[F, Binary[F]] = data .through(fs2.io.toInputStream[F]) .flatMap(in => unzipJava(in, chunkSize, glob)) + def saveTo[F[_]: Async]( + logger: Logger[F], + targetDir: Path, + moveUp: Boolean + ): Pipe[F, Binary[F], Path] = + binaries => + binaries + .filter(e => !e.name.endsWith("/")) + .evalMap { entry => + val out = targetDir / entry.name + val createParent = + OptionT + .fromOption[F](out.parent) + .flatMapF(parent => + Files[F] + .exists(parent) + .map(flag => Option.when(!flag)(parent)) + ) + .semiflatMap(p => Files[F].createDirectories(p)) + .getOrElse(()) + + logger.trace(s"Unzip ${entry.name} -> $out") *> + createParent *> + entry.data.through(Files[F].writeAll(out)).compile.drain + } + .drain ++ Stream + .eval(if (moveUp) moveContentsUp(logger)(targetDir) else ().pure[F]) + .as(targetDir) + + private def moveContentsUp[F[_]: Sync: Files](logger: Logger[F])(dir: Path): F[Unit] = + Files[F] + .list(dir) + .take(2) + .compile + .toList + .flatMap { + case subdir :: Nil => + Files[F].isDirectory(subdir).flatMap { + case false => ().pure[F] + case true => + Files[F] + .list(subdir) + .filter(p => p != dir) + .evalTap(c => logger.trace(s"Move $c -> ${dir / c.fileName}")) + .evalMap(child => Files[F].move(child, dir / child.fileName)) + .compile + .drain + } + + case _ => + ().pure[F] + } + def unzipJava[F[_]: Async]( in: InputStream, chunkSize: Int, @@ -55,7 +112,7 @@ object Zip { .unNoneTerminate .filter(ze => glob.matchFilenameOrPath(ze.getName())) .map { ze => - val name = Paths.get(ze.getName()).getFileName.toString + val name = ze.getName() val data = fs2.io.readInputStream[F]((zin: InputStream).pure[F], chunkSize, false) Binary(name, data) diff --git a/modules/files/src/test/resources/zip-dirs-one.zip b/modules/files/src/test/resources/zip-dirs-one.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a68b33aed12473ef2ba6f817e5c32ecd82b3d4e GIT binary patch literal 1306 zcmajfK}y3w6b9f)lZZx;fExu77orQHwNYHS5picX5i0HkwG|5yMIux;B0Yo`5O3iT z1drjyJNW*YO!|_UF)5Q~k@@o8%TLmtA4CGj`o&vcUKFQLtcd6Q_P%r4pA-Jk_thoV z<9J?p!H+LQFrXXLa{Fd&d!)AyMh{(O9MgU*r?FKt;)(&NkWNU!vPvzg3)0&5{@x5- zV=Y^x>rpn`Y3JiyHs~O4hFNwOsWt(lO%5+sV30{kI{M`#iK1QWnollKlRKkcU;gVC zhO4ZkgR*wZvVdXLhR4f7r^o{*7j&o4x~1j_hSt2+s_)9z61t}&7Qcu1FVYsOwAIQ> zdsE8H8mA64RV0r*YwD(+nLF-HDxOXusn!H4)t*76dgQ&oepytO$Y$P5BIh^gDRQ22 z2rQS)(=hYMr}4yKq?0^LnDpDi9L#L;LO&}WOKLN0CX~Z5JzGwI<&{k9M=ZoQLd??d KgLa*xYVijgEA^!S literal 0 HcmV?d00001 diff --git a/modules/files/src/test/resources/zip-dirs.zip b/modules/files/src/test/resources/zip-dirs.zip new file mode 100644 index 0000000000000000000000000000000000000000..816f8413ad768f4c3220a3f37d425386b03bb768 GIT binary patch literal 1098 zcmWIWW@h1H00H&H=^C#YQh*8r z;M%p(v~$C>gUo;$5(qQ^gs~dJ1T+q0h=PHY85@ud!Xl`Kq-Ex$8tau*lz^QCGN~TZ zw2ai8oO~{b!$4+&{Do{LAIwaUX-06fjb6=Ip|(fK{JgL=rxc*hH!(Nu{#K65F?WuGp^W|0C^V(1Q^~rf@o-3V1=Xw zj3`7Hg_#nNjk1Rs1xpt|1F@zFh=JHs1;ju`pixU2TTl%|N+X0##!MT?Cf|UWjFeIc zdml4}AbVd8=xCtNLFol(6eyl?ge1u0n5hNXd8sg?kkSreGcnT)vYDHKW+J5^pmCrS WgyBS1Q08P{2SN*=JO6+pf&l=Ev&Ap~ literal 0 HcmV?d00001 diff --git a/modules/files/src/test/scala/docspell/files/ZipTest.scala b/modules/files/src/test/scala/docspell/files/ZipTest.scala index 6012e1c2..9230ac8f 100644 --- a/modules/files/src/test/scala/docspell/files/ZipTest.scala +++ b/modules/files/src/test/scala/docspell/files/ZipTest.scala @@ -7,20 +7,25 @@ package docspell.files import cats.effect._ -import cats.effect.unsafe.implicits.global import cats.implicits._ +import fs2.io.file.{Files, Path} import docspell.common.Glob +import docspell.logging.TestLoggingConfig import munit._ -class ZipTest extends FunSuite { +class ZipTest extends CatsEffectSuite with TestLoggingConfig { + val logger = docspell.logging.getLogger[IO] + val tempDir = ResourceFixture( + Files[IO].tempDirectory(Path("target").some, "zip-test-", None) + ) test("unzip") { val zipFile = ExampleFiles.letters_zip.readURL[IO](8192) - val uncomp = zipFile.through(Zip.unzip(8192, Glob.all)) + val unzip = zipFile.through(Zip.unzip(8192, Glob.all)) - uncomp + unzip .evalMap { entry => val x = entry.data.map(_ => 1).foldMonoid.compile.lastOrError x.map { size => @@ -35,6 +40,10 @@ class ZipTest extends FunSuite { } .compile .drain - .unsafeRunSync() + } + + tempDir.test("unzipTo directory tree") { _ => + // val zipFile = ExampleFiles.zip_dirs_zip.readURL[IO](8192) + // zipFile.through(Zip.unzip(G)) } } diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 780d48b4..7a7d9b97 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -780,4 +780,75 @@ Docpell Update Check index-all-chunk = 10 } } + + addons { + # A directory to extract addons when running them. Everything in + # here will be cleared after each run. + working-dir = ${java.io.tmpdir}"/docspell-addons" + + # A directory for addons to store data between runs. This is not + # cleared by Docspell and can get large depending on the addons + # executed. + # + # This directory is used as base. In it subdirectories are created + # per run configuration id. + cache-dir = ${java.io.tmpdir}"/docspell-addon-cache" + + executor-config { + # Define a (comma or whitespace separated) list of runners that + # are responsible for executing an addon. This setting is + # compared to what is supported by addons. Possible values are: + # + # - nix-flake: use nix-flake runner if the addon supports it + # (this requires the nix package manager on the joex machine) + # - docker: use docker + # - trivial: use the trivial runner + # + # The first successful execution is used. This should list all + # runners the computer supports. + runner = "nix-flake, docker, trivial" + + # systemd-nspawn can be used to run the program in a container. + # This is used by runners nix-flake and trivial. + nspawn = { + # If this is false, systemd-nspawn is not tried. When true, the + # addon is executed inside a lightweight container via + # systemd-nspawn. + enabled = false + + # Path to sudo command. By default systemd-nspawn is executed + # via sudo - the user running joex must be allowed to do so NON + # INTERACTIVELY. If this is empty, then nspawn is tried to + # execute without sudo. + sudo-binary = "sudo" + + # Path to the systemd-nspawn command. + nspawn-binary = "systemd-nspawn" + + # Workaround, if multiple same named containers are run too fast + container-wait = "100 millis" + } + + # The timeout for running an addon. + run-timeout = "15 minutes" + + # Configure the nix flake runner. + nix-runner { + # Path to the nix command. + nix-binary = "nix" + + # The timeout for building the package (running nix build). + build-timeout = "15 minutes" + } + + # Configure the docker runner + docker-runner { + # Path to the docker command. + docker-binary = "docker" + + # The timeout for building the package (running docker build). + build-timeout = "15 minutes" + } + } + } } \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index de171135..646033c2 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.TextAnalysisConfig import docspell.analysis.classifier.TextClassifierConfig import docspell.backend.Config.Files +import docspell.backend.joex.AddonEnvConfig import docspell.common._ import docspell.config.{FtsType, PgFtsConfig} import docspell.convert.ConvertConfig @@ -43,7 +44,8 @@ case class Config( files: Files, mailDebug: Boolean, fullTextSearch: Config.FullTextSearch, - updateCheck: UpdateCheckConfig + updateCheck: UpdateCheckConfig, + addons: AddonEnvConfig ) { def pubSubConfig(headerValue: Ident): PubSubConfig = diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index c410cc42..e3a679a8 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -145,6 +145,8 @@ object JoexAppImpl extends MailAddressCodec { schedulerModule.scheduler, schedulerModule.periodicScheduler ) + nodes <- ONode(store) + _ <- nodes.withRegistered(cfg.appId, NodeType.Joex, cfg.baseUrl, None) appR <- Resource.make(app.init.map(_ => app))(_.initShutdown) } yield appR diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 5bdc8f18..ae1bf1a7 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -59,7 +59,7 @@ object JoexServer { Router("pubsub" -> pubSub.receiveRoute) }, "/api/info" -> InfoRoutes(cfg), - "/api/v1" -> JoexRoutes(joexApp) + "/api/v1" -> JoexRoutes(cfg, joexApp) ).orNotFound // With Middlewares in place diff --git a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala index 913ee930..f5c99d47 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala @@ -9,7 +9,9 @@ package docspell.joex import cats.effect.{Async, Resource} import docspell.analysis.TextAnalyser +import docspell.backend.BackendCommands import docspell.backend.fulltext.CreateIndex +import docspell.backend.joex.AddonOps import docspell.backend.ops._ import docspell.backend.task.DownloadZipArgs import docspell.common._ @@ -17,6 +19,7 @@ import docspell.config.FtsType import docspell.ftsclient.FtsClient import docspell.ftspsql.PsqlFtsClient import docspell.ftssolr.SolrFtsClient +import docspell.joex.addon.{ItemAddonTask, ScheduledAddonTask} import docspell.joex.analysis.RegexNerFile import docspell.joex.download.DownloadZipTask import docspell.joex.emptytrash.EmptyTrashTask @@ -32,6 +35,7 @@ import docspell.joex.preview.{AllPreviewsTask, MakePreviewTask} import docspell.joex.process.{ItemHandler, ReProcessItem} import docspell.joex.scanmailbox.ScanMailboxTask import docspell.joex.updatecheck.{ThisVersion, UpdateCheck, UpdateCheckTask} +import docspell.joexapi.client.JoexClient import docspell.notification.api.NotificationModule import docspell.pubsub.api.PubSubT import docspell.scheduler.impl.JobStoreModuleBuilder @@ -57,7 +61,8 @@ final class JoexTasks[F[_]: Async]( createIndex: CreateIndex[F], joex: OJoex[F], jobs: OJob[F], - itemSearch: OItemSearch[F] + itemSearch: OItemSearch[F], + addons: AddonOps[F] ) { val downloadAll: ODownloadAll[F] = ODownloadAll(store, jobs, jobStoreModule.jobs) @@ -68,7 +73,8 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, store, itemOps, fts, analyser, regexNer), + ItemHandler + .newItem[F](cfg, store, itemOps, fts, analyser, regexNer, addons), ItemHandler.onCancel[F](store) ) ) @@ -82,7 +88,15 @@ final class JoexTasks[F[_]: Async]( .withTask( JobTask.json( ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), + ReProcessItem[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addons, + store + ), ReProcessItem.onCancel[F] ) ) @@ -223,6 +237,20 @@ final class JoexTasks[F[_]: Async]( DownloadZipTask.onCancel[F] ) ) + .withTask( + JobTask.json( + ScheduledAddonTaskArgs.taskName, + ScheduledAddonTask[F](addons), + ScheduledAddonTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ItemAddonTaskArgs.taskName, + ItemAddonTask[F](addons, store), + ItemAddonTask.onCancel[F] + ) + ) } object JoexTasks { @@ -237,8 +265,9 @@ object JoexTasks { emailService: Emil[F] ): Resource[F, JoexTasks[F]] = for { - joex <- OJoex(pubSub) - store = jobStoreModule.store + store <- Resource.pure(jobStoreModule.store) + node <- ONode(store) + joex <- OJoex(pubSub, node, JoexClient(httpClient)) upload <- OUpload(store, jobStoreModule.jobs) fts <- createFtsClient(cfg, pools, store, httpClient) createIndex <- CreateIndex.resource(fts, store) @@ -250,6 +279,16 @@ object JoexTasks { notification <- ONotification(store, notificationModule) fileRepo <- OFileRepository(store, jobStoreModule.jobs) jobs <- OJob(store, joex, pubSub) + fields <- OCustomFields(store) + attachmentOps = OAttachment(store, fts, jobStoreModule.jobs) + cmdRunner = BackendCommands(itemOps, attachmentOps, fields, notification, None) + addons = AddonOps( + cfg.addons, + store, + cmdRunner, + attachmentOps, + jobStoreModule.jobs + ) } yield new JoexTasks[F]( cfg, store, @@ -266,7 +305,8 @@ object JoexTasks { createIndex, joex, jobs, - itemSearchOps + itemSearchOps, + addons ) private def createFtsClient[F[_]: Async]( diff --git a/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala new file mode 100644 index 00000000..0b1bad57 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/AddonTaskExtension.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.MonadError + +import docspell.addons.AddonExecutionResult +import docspell.scheduler.PermanentError + +trait AddonTaskExtension { + implicit final class AddonExecutionResultOps(self: AddonExecutionResult) { + def raiseErrorIfNeeded[F[_]](implicit m: MonadError[F, Throwable]): F[Unit] = + if (self.isFailure && self.pure) { + m.raiseError(new Exception(s"Addon execution failed: $self")) + } else if (self.isFailure) { + m.raiseError( + PermanentError( + new Exception( + "Addon execution failed. Do not retry, because some addons were impure." + ) + ) + ) + } else m.pure(()) + + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala new file mode 100644 index 00000000..50bbf41b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.{Kleisli, OptionT} +import cats.effect._ +import cats.syntax.all._ +import fs2.io.file.Files + +import docspell.addons.{AddonTriggerType, InputEnv, Middleware} +import docspell.backend.joex.AddonOps.ExecResult +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.common._ +import docspell.files.FileSupport +import docspell.joex.process.ItemData +import docspell.logging.Logger +import docspell.scheduler.Task +import docspell.store.Store +import docspell.store.queries.QAttachment + +object GenericItemAddonTask extends LoggerExtension with FileSupport { + + private val itemSubdir = "item" + private val itemDataJson = s"$itemSubdir/item-data.json" + private val argsMetaJson = s"$itemSubdir/given-data.json" + private val pdfDir = s"$itemSubdir/pdfs" + private val originalDir = s"$itemSubdir/originals" + private val originalMetaJson = s"$itemSubdir/source-files.json" + private val pdfMetaJson = s"$itemSubdir/pdf-files.json" + + // This environment can be used by the addon to access data of the current task + private val itemEnv = Map( + "ITEM_DIR" -> itemSubdir, + "ITEM_DATA_JSON" -> itemDataJson, + "ITEM_ARGS_JSON" -> argsMetaJson, + "ITEM_PDF_DIR" -> pdfDir, + "ITEM_ORIGINAL_DIR" -> originalDir, + "ITEM_ORIGINAL_JSON" -> originalMetaJson, + "ITEM_PDF_JSON" -> pdfMetaJson + ) + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ItemData] = + addonResult(ops, store, trigger, addonTaskIds)(collective, data, maybeMeta).as( + data + ) + + def addonResult[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + )( + collective: Ident, + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + ): Task[F, Unit, ExecResult] = + Task { ctx => + ops.execAll(collective, Set(trigger), addonTaskIds, ctx.logger.some)( + Middleware.prepare(Kleisli(prepareItemData(ctx.logger, store, data, maybeMeta))) + ) + } + + def prepareItemData[F[_]: Async]( + logger: Logger[F], + store: Store[F], + data: ItemData, + maybeMeta: Option[ProcessItemArgs.ProcessMeta] + )( + input: InputEnv + ): F[InputEnv] = + for { + _ <- logger.debug(s"Preparing item data '${data.item.name}' for addon") + wd = input.baseDir + itemMetaFile = wd / itemDataJson + argsMetaFile = wd / argsMetaJson + pdfs = wd / pdfDir + originals = wd / originalDir + srcJson = wd / originalMetaJson + pdfJson = wd / pdfMetaJson + + _ <- List(wd / itemSubdir, pdfs, originals).traverse(Files[F].createDirectories) + + _ <- logger.debug("Writing collected item data…") + _ <- itemMetaFile.writeJson(data) + + _ <- OptionT + .fromOption[F](maybeMeta) + .semiflatMap { meta => + logger.debug("Writing context meta data…") *> + argsMetaFile.writeJson(meta) + } + .value + + _ <- logger.debug("Storing all attachments…") + _ <- data.attachments + .flatMap(a => + Vector( + pdfs / a.id.id -> a.fileId, + originals / a.id.id -> data.originFile(a.id) + ) + ) + .traverse_ { case (out, key) => + logger.debug(s"Storing attachment $out") *> + store.fileRepo + .getBytes(key) + .through(Files[F].writeAll(out)) + .compile + .drain + } + + _ <- logger.debug("Storing file metadata") + srcMeta <- store.transact(QAttachment.attachmentSourceFile(data.item.id)) + pdfMeta <- store.transact(QAttachment.attachmentFile(data.item.id)) + _ <- srcJson.writeJson(srcMeta) + _ <- pdfJson.writeJson(pdfMeta) + } yield input.addEnv(itemEnv) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala new file mode 100644 index 00000000..32f8bc7b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ItemAddonTask.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.{ItemAddonTaskArgs, MetaProposalList} +import docspell.joex.process.ItemData +import docspell.scheduler.{PermanentError, Task} +import docspell.store.Store +import docspell.store.queries.QAttachment +import docspell.store.records._ + +object ItemAddonTask extends AddonTaskExtension { + type Args = ItemAddonTaskArgs + val name = ItemAddonTaskArgs.taskName + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + + def apply[F[_]: Async](ops: AddonOps[F], store: Store[F]): Task[F, Args, Result] = + Task { ctx => + (for { + item <- OptionT( + store.transact( + RItem.findByIdAndCollective(ctx.args.itemId, ctx.args.collective) + ) + ) + data <- OptionT.liftF(makeItemData(store, item)) + inner = GenericItemAddonTask.addonResult( + ops, + store, + AddonTriggerType.ExistingItem, + ctx.args.addonRunConfigs + )(ctx.args.collective, data, None) + execResult <- OptionT.liftF(inner.run(ctx.unit)) + _ <- OptionT.liftF(execResult.combined.raiseErrorIfNeeded[F]) + } yield Result( + execResult.combined.addonResult, + execResult.runConfigs.flatMap(_.refs).map(_.archive.nameAndVersion).distinct + )).getOrElseF( + Async[F].raiseError( + PermanentError( + new NoSuchElementException(s"Item not found for id: ${ctx.args.itemId.id}!") + ) + ) + ) + } + + def makeItemData[F[_]: Async](store: Store[F], item: RItem): F[ItemData] = + for { + attachs <- store.transact(RAttachment.findByItem(item.id)) + rmeta <- store.transact(QAttachment.getAttachmentMetaOfItem(item.id)) + rsource <- store.transact(RAttachmentSource.findByItem(item.id)) + proposals <- store.transact(QAttachment.getMetaProposals(item.id, item.cid)) + tags <- store.transact(RTag.findByItem(item.id)) + } yield ItemData( + item = item, + attachments = attachs, + metas = rmeta, + dateLabels = Vector.empty, + originFile = rsource.map(r => (r.id, r.fileId)).toMap, + givenMeta = proposals, + tags = tags.map(_.name).toList, + classifyProposals = MetaProposalList.empty, + classifyTags = Nil + ) +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/Result.scala b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala new file mode 100644 index 00000000..fdd86ba2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/Result.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import docspell.addons.AddonResult +import docspell.scheduler.JobTaskResultEncoder + +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder + +case class Result(addonResult: AddonResult, addons: List[String]) + +object Result { + val empty: Result = + Result(AddonResult.empty, Nil) + + implicit val jsonEncoder: Encoder[Result] = + deriveEncoder + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[Result] = + JobTaskResultEncoder.fromJson[Result].withMessage { result => + result.addonResult match { + case AddonResult.Success(_) => + s"Executed ${result.addons.size} addon(s) successfully." + + case AddonResult.ExecutionError(rc) => + s"Addon execution finished with non-zero return code: $rc" + + case AddonResult.ExecutionFailed(ex) => + s"Addon execution failed: ${ex.getMessage}" + + case AddonResult.DecodingError(msg) => + s"Addon output failed to read: $msg" + } + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala new file mode 100644 index 00000000..7f108223 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/addon/ScheduledAddonTask.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.addon + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.Middleware +import docspell.backend.joex.{AddonOps, LoggerExtension} +import docspell.backend.ops.OAddons +import docspell.common.{Ident, ScheduledAddonTaskArgs} +import docspell.scheduler.Task + +object ScheduledAddonTask extends AddonTaskExtension with LoggerExtension { + type Args = ScheduledAddonTaskArgs + + val name: Ident = OAddons.scheduledAddonTaskName + + def apply[F[_]: Async](ops: AddonOps[F]): Task[F, Args, Result] = + Task { ctx => + for { + execRes <- ops.execById(ctx.args.collective, ctx.args.addonTaskId, ctx.logger)( + Middleware.identity[F] + ) + _ <- execRes.result.combineAll.raiseErrorIfNeeded[F] + } yield Result( + execRes.result.combineAll.addonResult, + execRes.runConfigs.flatMap(_.refs.map(_.archive.nameAndVersion)) + ) + } + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${name.id} task")) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala index 4e6d9f52..ea0500c6 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/NerFile.scala @@ -12,6 +12,7 @@ import fs2.io.file.Path import docspell.analysis.split.TextSplitter import docspell.common._ +import docspell.common.util.File import docspell.store.queries.QCollective import io.circe.generic.semiauto._ diff --git a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala index 2f46234e..8d3f3562 100644 --- a/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.io.file.Path import docspell.common._ +import docspell.common.util.File import docspell.store.Store import docspell.store.queries.QCollective import docspell.store.records.REquipment diff --git a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala index 1d61f4fd..24244d6b 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/Classify.scala @@ -14,6 +14,7 @@ import fs2.io.file.Path import docspell.analysis.classifier.{ClassifierModel, TextClassifier} import docspell.common._ +import docspell.common.util.File import docspell.logging.Logger import docspell.store.Store import docspell.store.records.RClassifierModel diff --git a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala index 6fc5f634..0470fcdc 100644 --- a/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala @@ -109,7 +109,7 @@ object MultiUploadArchiveTask { )(file: ProcessItemArgs.File): Stream[F, ProcessItemArgs] = store.fileRepo .getBytes(file.fileMetaId) - .through(Zip.unzipP[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) + .through(Zip.unzip[F](8192, args.meta.fileFilter.getOrElse(Glob.all))) .flatMap { entry => val hint = MimeTypeHint(entry.name.some, entry.mime.asString.some) entry.data diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index 31c1e007..98a2923b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -69,7 +69,7 @@ object AttachmentPreview { } case mt => - ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot get page count.") *> + ctx.logger.warn(s"Not a pdf file, but ${mt.asString}, cannot create preview.") *> (None: Option[RAttachmentPreview]).pure[F] } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index 5e2d86b0..27cbe414 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -146,7 +146,7 @@ object ExtractArchive { val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all) ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *> zipData - .through(Zip.unzipP[F](8192, glob)) + .through(Zip.unzip[F](8192, glob)) .zipWithIndex .flatMap(handleEntry(ctx, store, ra, pos, archive, None)) .foldMonoid diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index c96c0189..94a6c07f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -26,7 +26,8 @@ import io.circe.{Encoder, Json} * @param dateLabels * a separate list of found dates * @param originFile - * a mapping from an attachment id to a filemeta-id containng the source or origin file + * a mapping from an attachment id to a filemeta-id containing the source or origin + * file * @param givenMeta * meta data to this item that was not "guessed" from an attachment but given and thus * is always correct @@ -49,7 +50,7 @@ case class ItemData( ) { /** sort by weight; order of equal weights is not important, just choose one others are - * then suggestions doc-date is only set when given explicitely, not from "guessing" + * then suggestions doc-date is only set when given explicitly, not from "guessing" */ def finalProposals: MetaProposalList = MetaProposalList @@ -98,7 +99,7 @@ object ItemData { dates.map(dl => dl.label.copy(label = dl.date.toString)) } - // Used to encode the result passed to the job-done event + // Used to encode the result passed to the job-done event and to supply to addons implicit val jsonEncoder: Encoder[ItemData] = Encoder.instance { data => val metaMap = data.metas.groupMap(_.id)(identity) @@ -108,10 +109,12 @@ object ItemData { "collective" -> data.item.cid.asJson, "source" -> data.item.source.asJson, "attachments" -> data.attachments + .sortBy(_.position) .map(a => Json.obj( "id" -> a.id.asJson, "name" -> a.name.asJson, + "position" -> a.position.asJson, "content" -> metaMap.get(a.id).flatMap(_.head.content).asJson, "language" -> metaMap.get(a.id).flatMap(_.head.language).asJson, "pages" -> metaMap.get(a.id).flatMap(_.head.pages).asJson @@ -123,6 +126,18 @@ object ItemData { "assumedCorrOrg" -> data.finalProposals .find(MetaProposalType.CorrOrg) .map(_.values.head.ref) + .asJson, + "assumedCorrPerson" -> data.finalProposals + .find(MetaProposalType.CorrPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcPerson" -> data.finalProposals + .find(MetaProposalType.ConcPerson) + .map(_.values.head.ref) + .asJson, + "assumedConcEquip" -> data.finalProposals + .find(MetaProposalType.ConcEquip) + .map(_.values.head.ref) .asJson ) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 8d59a969..b5fc216e 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -12,6 +12,7 @@ import cats.implicits._ import fs2.Stream import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} import docspell.ftsclient.FtsClient @@ -41,7 +42,8 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] ): Task[F, Args, Option[ItemData]] = logBeginning[F].flatMap(_ => DuplicateCheck[F](store) @@ -52,7 +54,17 @@ object ItemHandler { CreateItem[F](store).contramap(_ => args.pure[F]) create .flatMap(itemStateTask(store, ItemState.Processing)) - .flatMap(safeProcess[F](cfg, store, itemOps, fts, analyser, regexNer)) + .flatMap( + safeProcess[F]( + cfg, + store, + itemOps, + fts, + analyser, + regexNer, + addons + ) + ) .map(_.some) } ) @@ -76,11 +88,14 @@ object ItemHandler { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + addons: AddonOps[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data).attempt + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -91,7 +106,9 @@ object ItemHandler { .andThen(_ => Sync[F].raiseError(ex)) } case false => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data) + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, addons, store)( + data + ) .flatMap(itemStateTask(store, ItemState.Created)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 6087b37f..836a8062 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -9,7 +9,9 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs import docspell.ftsclient.FtsClient @@ -26,6 +28,7 @@ object ProcessItem { fts: FtsClient[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = ExtractArchive(store)(item) @@ -35,6 +38,7 @@ object ProcessItem { .flatMap(SetGivenData.onlyNew[F](itemOps)) .flatMap(Task.setProgress(99)) .flatMap(RemoveEmptyItem(itemOps)) + .flatMap(RunAddons(addonOps, store, AddonTriggerType.FinalProcessItem)) def processAttachments[F[_]: Async]( cfg: Config, diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 1863d2ef..6890e37d 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -10,7 +10,9 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ +import docspell.addons.AddonTriggerType import docspell.analysis.TextAnalyser +import docspell.backend.joex.AddonOps import docspell.backend.ops.OItem import docspell.common._ import docspell.ftsclient.FtsClient @@ -34,13 +36,24 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] ): Task[F, Args, Unit] = Task .log[F, Args](_.info("===== Start reprocessing ======")) .flatMap(_ => loadItem[F](store) - .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer, store)) + .flatMap( + safeProcess[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store + ) + ) .map(_ => ()) ) @@ -99,6 +112,7 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F], data: ItemData ): Task[F, Args, ItemData] = { @@ -129,6 +143,7 @@ object ReProcessItem { .processAttachments[F](cfg, fts, analyser, regexNer, store)(data) .flatMap(LinkProposal[F](store)) .flatMap(SetGivenData[F](itemOps)) + .flatMap(RunAddons[F](addonOps, store, AddonTriggerType.FinalReprocessItem)) .contramap[Args](convertArgs(lang)) } } @@ -153,11 +168,21 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + addonOps: AddonOps[F], store: Store[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data).attempt + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ).attempt .flatMap { case Right(d) => Task.pure(d) @@ -167,7 +192,16 @@ object ReProcessItem { ).andThen(_ => Sync[F].raiseError(ex)) } case false => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data) + processFiles[F]( + cfg, + fts, + itemOps, + analyser, + regexNer, + addonOps, + store, + data + ) } private def logWarn[F[_]](msg: => String): Task[F, Args, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala new file mode 100644 index 00000000..b564d8b4 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/RunAddons.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.process + +import cats.effect._ +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.backend.joex.AddonOps +import docspell.common.ProcessItemArgs +import docspell.joex.addon.GenericItemAddonTask +import docspell.scheduler.Task +import docspell.store.Store + +/** Run registered addons in the context of item processing. The addon has access to the + * current item data and can apply custom processing logic. + */ +object RunAddons { + type Args = ProcessItemArgs + + def apply[F[_]: Async]( + ops: AddonOps[F], + store: Store[F], + trigger: AddonTriggerType + )( + data: ItemData + ): Task[F, Args, ItemData] = + if (data.item.state.isInvalid && data.attachments.isEmpty) { + Task.pure(data) + } else + Task { ctx => + val inner = GenericItemAddonTask(ops, store, trigger, Set.empty)( + ctx.args.meta.collective, + data, + ctx.args.meta.some + ) + inner.run(ctx.unit) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala index 0734d294..7fd35abe 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -44,7 +44,7 @@ object SetGivenData { for { _ <- ctx.logger.info("Starting setting given data") _ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'") - e <- ops.setFolder(itemId, folderId, collective).attempt + e <- ops.setFolder(itemId, folderId.map(_.id), collective).attempt _ <- e.fold( ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"), res => diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala index 6810f3d9..e02ff31d 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Duration, Ident, Timestamp} -import docspell.joex.JoexApp +import docspell.joex.{Config, JoexApp} import docspell.joexapi.model._ import docspell.store.records.RJobLog @@ -20,7 +20,7 @@ import org.http4s.dsl.Http4sDsl object JoexRoutes { - def apply[F[_]: Async](app: JoexApp[F]): HttpRoutes[F] = { + def apply[F[_]: Async](cfg: Config, app: JoexApp[F]): HttpRoutes[F] = { val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { @@ -64,6 +64,11 @@ object JoexRoutes { BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found") ) } yield resp + + case GET -> Root / "addon" / "config" => + val data = + AddonSupport(cfg.appId, cfg.addons.executorConfig.runner) + Ok(data) } } diff --git a/modules/joexapi/src/main/resources/joex-openapi.yml b/modules/joexapi/src/main/resources/joex-openapi.yml index 27bb62eb..c0e1f55a 100644 --- a/modules/joexapi/src/main/resources/joex-openapi.yml +++ b/modules/joexapi/src/main/resources/joex-openapi.yml @@ -122,8 +122,45 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /addon/config: + get: + operationId: "v1-addon-config-get" + tags: [ Addons ] + summary: What is supported running addons + description: | + Return what this joex supports when executing addons. + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonSupport" + components: schemas: + AddonSupport: + description: | + How this joex supports executing addons. + required: + - nodeId + - runners + properties: + nodeId: + type: string + format: ident + runners: + type: array + items: + type: string + format: addon-runner-type + enum: + - nix-flake + - docker + - trivial + JobAndLog: description: | Some more details about the job. diff --git a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala index c623eba0..b9b5b5ef 100644 --- a/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala +++ b/modules/joexapi/src/main/scala/docspell/joexapi/client/JoexClient.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common.{Ident, LenientUri} -import docspell.joexapi.model.BasicResult +import docspell.joexapi.model.{AddonSupport, BasicResult} import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.circe.CirceEntityDecoder @@ -25,6 +25,7 @@ trait JoexClient[F[_]] { def cancelJob(base: LenientUri, job: Ident): F[BasicResult] + def getAddonSupport(base: LenientUri): F[AddonSupport] } object JoexClient { @@ -33,6 +34,13 @@ object JoexClient { new JoexClient[F] with CirceEntityDecoder { private[this] val logger = docspell.logging.getLogger[F] + def getAddonSupport(base: LenientUri): F[AddonSupport] = { + val getUrl = base / "api" / "v1" / "addon" / "config" + val req = Request[F](Method.GET, uri(getUrl)) + logger.debug(s"Getting addon support") *> + client.expect[AddonSupport](req) + } + def notifyJoex(base: LenientUri): F[BasicResult] = { val notifyUrl = base / "api" / "v1" / "notify" val req = Request[F](Method.POST, uri(notifyUrl)) diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala index 83f320bb..f5aaee15 100644 --- a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala +++ b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala @@ -143,7 +143,9 @@ object Event { } - /** Some generic list of items, chosen by a user. */ + /** Some generic list of items, chosen by a user. This is use to notify about periodic + * search results. + */ final case class ItemSelection( account: AccountId, items: Nel[Ident], diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 7a509842..ab9cc8a9 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -5790,9 +5790,388 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /sec/addon/archive: + get: + operationId: "sec-addon-archive-get" + tags: [Addons] + summary: Get all registered addons + description: | + Returns a list of all registered addons. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonList" + post: + operationId: "sec-addon-post" + tags: [ Addons ] + summary: Install a new addon + description: | + Given an URL to an addon (which is a zip file containing a + `docspell-meta.yaml` or json descriptor), the addon is + downloaded and installed in docspell. + + By default this happens asynchronously and the response only + indicates that installing has been submitted. The result will + be transfered over the websocket channel. With query parameter + `sync` installing happens synchronously and it may take a + while to complete (if successful, the addon id is returned). + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRegister" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + /sec/addon/archive/{id}: + parameters: + - $ref: "#/components/parameters/id" + delete: + operationId: "sec-addon-archive-delete" + tags: [Addons] + summary: Deletes the addon and removes it from all addon run configs + description: | + Deletes the addon from the database and also removes it from + all run configurations where it might be referenced. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + operationId: "sec-addon-archive-put" + tags: [ Addons ] + summary: Update an addon from its url + description: | + Addons are urls to zip files. This call reads the url again + and updates the contents in docspell for this addon. + + By default this happens asynchronously and the response only + indicates that updating has been submitted. The result will be + transfered over the websocket channel. With query parameter + `sync` updating happens synchronously and it may take a while + to complete. + security: + - authTokenHeader: [] + parameters: + - in: query + name: sync + required: false + allowEmptyValue: true + schema: + type: boolean + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run-config: + get: + operationId: "sec-addon-run-config-get" + tags: [Addons] + summary: Get all addon run configs + description: | + Returns a list of addon run configs. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfigList" + post: + operationId: "sec-addon-run-config-post" + tags: [ Addons ] + summary: Adds a new addon run config + description: | + Adds a new set of configured addons, creating a run + configuration. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/IdResult" + + /sec/addon/run-config/{id}: + parameters: + - $ref: "#/components/parameters/id" + put: + operationId: "sec-addon-run-config-id-put" + tags: [ Addons ] + summary: Updates an addon run config + description: | + Updates an existing addon run configuration. The id is taken + from the URL and any given id in the request body is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunConfig" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + delete: + operationId: "sec-addonrunconfig-delete" + tags: [Addons] + summary: Deletes the addon run config given its id + description: | + Deletes the addon run configuration. + security: + - authTokenHeader: [] + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + + /sec/addon/run/existingitem: + post: + operationId: "sec-addon-run-existing-item" + tags: [Addons] + summary: Submits a task running addons for an item + description: | + Submits a background task that executes the specified (or all) + addons configured to use for an existing item. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AddonRunExistingItem" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" components: schemas: + AddonRunExistingItem: + description: | + Data to run addons for an existing item. + required: + - itemId + properties: + itemId: + type: string + format: ident + additionalItems: + type: array + items: + type: string + format: ident + description: | + Additional items to run addons on. There will be one job + submitted per item. + addonRunConfigIds: + type: array + items: + type: string + format: ident + description: | + If non empty, only select these addon run configs. + Otherwise all configured to be run for existing items are + executed. + + AddonRef: + description: | + A reference to an addon (archive) with additional name and + version and its arguments. When used for adding addon run + configs, name and version are ignored. + required: + - addonId + - name + - version + - args + properties: + addonId: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + args: + type: string + + AddonRunConfig: + description: | + A set of configured addons that are run on certain points + defined by the `trigger` property. + required: + - id + - name + - enabled + - trigger + - addons + properties: + id: + type: string + format: ident + name: + type: string + enabled: + type: boolean + userId: + type: string + format: ident + description: | + An addon can be run on behalf of a user. If not given, no + authentication token is generated into the environment of + the addon. The user can be given as user_id or by its + login name. + schedule: + type: string + format: calevent + description: | + A schedule must be supplied when a trigger type of + 'scheduled' is defined. + trigger: + description: | + Defines when this task is executed. There must be at least + one element. Possible values: + + * process-item: After an item has been processed + * reprocess-item: After an item has been re-processed + * scheduled: Executed periodically based on a schedule, + which must be defined then + type: array + items: + type: string + format: addon-trigger-type + enum: + - process-item + - reprocess-item + - scheduled + addons: + type: array + items: + $ref: "#/components/schemas/AddonRef" + + AddonRunConfigList: + description: | + A list of addon run configurations. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/AddonRunConfig" + + AddonRegister: + description: | + Data to register addons + required: + - url + properties: + url: + type: string + format: uri + + Addon: + description: | + An registered addon. + required: + - id + - name + - version + - created + properties: + id: + type: string + format: ident + name: + type: string + version: + type: string + description: + type: string + url: + type: string + format: uri + created: + type: integer + format: date-time + + AddonList: + description: | + A list of addons + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Addon" + DownloadAllSummary: description: | Information about a ZIP download. diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 508bb277..2c6d4523 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -462,5 +462,39 @@ docspell.server { } } } + + addons = { + enabled = false + + # Whether installing addons requiring network should be allowed + # or not. + allow-impure = true + + # Define patterns of urls that are allowed to install addons + # from. + # + # A pattern is compared against an URL by comparing three parts + # of an URL via globs: scheme, host and path. + # + # You can use '*' (0 or more) and '?' (one) as wildcards in each + # part. For example: + # + # https://*.mydomain.com/projects/* + # *s://gitea.mydomain/* + # + # A hostname is separated by dots and the path by a slash. A '*' + # in a pattern means to match one or more characters. The path + # pattern is always matching the given prefix. So /a/b/* matches + # /a/b/c and /a/b/c/d and all other sub-paths. + # + # Multiple patterns can be defined va a comma separated string + # or as an array. An empty string matches no URL, while the + # special pattern '*' all by itself means to match every URL. + allowed-urls = "*" + + # Same as `allowed-urls` but a match here means do deny addons + # from this url. + denied-urls = "" + } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 5d2b26a6..45f58c2e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -152,7 +152,9 @@ final class RestAppImpl[F[_]: Async]( "clientSettings" -> ClientSettingsRoutes(backend, token), "notification" -> NotificationRoutes(config, backend, token), "querybookmark" -> BookmarkRoutes(backend, token), - "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token) + "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token), + "addonrunconfig" -> AddonRunConfigRoutes(backend, token), + "addon" -> AddonRoutes(config, wsTopic, backend, token) ) } @@ -181,7 +183,16 @@ object RestAppImpl { .withEventSink(notificationMod) .build backend <- BackendApp - .create[F](store, javaEmil, ftsClient, pubSubT, schedulerMod, notificationMod) + .create[F]( + cfg.backend, + store, + javaEmil, + httpClient, + ftsClient, + pubSubT, + schedulerMod, + notificationMod + ) app = new RestAppImpl[F]( cfg, diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 1a8fd6c4..e4c1a5bc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -14,6 +14,7 @@ import fs2.Stream import fs2.concurrent.Topic import docspell.backend.msg.Topics +import docspell.backend.ops.ONode import docspell.common._ import docspell.pubsub.naive.NaivePubSub import docspell.restserver.http4s.InternalHeader @@ -91,6 +92,15 @@ object RestServer { store, httpClient )(Topics.all.map(_.topic)) + + nodes <- ONode(store) + _ <- nodes.withRegistered( + cfg.appId, + NodeType.Restserver, + cfg.baseUrl, + cfg.auth.serverSecret.some + ) + restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic) } yield (restApp, pubSub, setting) diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala new file mode 100644 index 00000000..dd2351b8 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.conv + +import cats.syntax.all._ + +import docspell.addons.AddonMeta +import docspell.backend.ops.AddonValidationError +import docspell.backend.ops.OAddons.AddonValidationResult +import docspell.common.Ident +import docspell.restserver.ws.{OutputEvent, OutputEventEncoder} +import docspell.store.records.RAddonArchive + +trait AddonValidationSupport { + + def validationErrorToMessage(e: AddonValidationError): String = + e match { + case AddonValidationError.AddonNotFound => + "Addon not found." + + case AddonValidationError.AddonExists(msg, _) => + msg + + case AddonValidationError.NotAnAddon(ex) => + s"The url doesn't seem to be an addon: ${ex.getMessage}" + + case AddonValidationError.InvalidAddon(msg) => + s"The addon is not valid: $msg" + + case AddonValidationError.AddonUnsupported(msg, _) => + msg + + case AddonValidationError.AddonsDisabled => + "Addons are disabled in the config file." + + case AddonValidationError.UrlUntrusted(_) => + "This url doesn't belong to te set of trusted urls defined in the config file" + + case AddonValidationError.DownloadFailed(ex) => + s"Downloading the addon failed: ${ex.getMessage}" + + case AddonValidationError.ImpureAddonsDisabled => + s"Installing impure addons is disabled." + + case AddonValidationError.RefreshLocalAddon => + "Refreshing a local addon doesn't work." + } + + def addonResultOutputEventEncoder( + collective: Ident + ): OutputEventEncoder[AddonValidationResult[(RAddonArchive, AddonMeta)]] = + OutputEventEncoder.instance { + case Right((archive, _)) => + OutputEvent.AddonInstalled( + collective, + "Addon installed", + None, + archive.id.some, + archive.originalUrl + ) + + case Left(error) => + val msg = validationErrorToMessage(error) + OutputEvent.AddonInstalled(collective, msg, error.some, None, None) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala index aead3504..6b17d49e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala @@ -47,4 +47,8 @@ object Responses { def notFoundRoute[F[_]: Sync]: HttpRoutes[F] = HttpRoutes(_ => OptionT.pure(Response.notFound[F])) + def notFoundRoute[F[_]: Sync, A](body: A)(implicit + entityEncoder: EntityEncoder[F, A] + ): HttpRoutes[F] = + HttpRoutes(_ => OptionT.pure(Response.notFound[F].withEntity(body))) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala new file mode 100644 index 00000000..ceeabadc --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.http4s + +import cats.effect._ + +import docspell.joexapi.model.BasicResult + +import org.http4s.Response +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +trait ThrowableResponseMapper { + + implicit class EitherThrowableOps[A](self: Either[Throwable, A]) { + def rightAs[F[_]: Sync](f: A => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], f) + + def rightAs_[F[_]: Sync](r: => F[Response[F]]): F[Response[F]] = + self.fold(ThrowableResponseMapper.toResponse[F], _ => r) + } +} + +object ThrowableResponseMapper { + def toResponse[F[_]: Sync](ex: Throwable): F[Response[F]] = + new Mapper[F].toResponse(ex) + + private class Mapper[F[_]: Sync] extends Http4sDsl[F] { + def toResponse(ex: Throwable): F[Response[F]] = + ex match { + case _: IllegalArgumentException => + BadRequest(BasicResult(false, ex.getMessage)) + + case _ => + InternalServerError(BasicResult(false, ex.getMessage)) + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala new file mode 100644 index 00000000..72c9b808 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.addons.AddonMeta +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.conv.AddonValidationSupport +import docspell.restserver.ws.{Background, OutputEvent} +import docspell.store.records.RAddonArchive + +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl +import org.http4s.dsl.impl.FlagQueryParamMatcher +import org.http4s.{HttpRoutes, Response} + +object AddonArchiveRoutes extends AddonValidationSupport { + + def apply[F[_]: Async]( + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + implicit val wsOutputEnc = addonResultOutputEventEncoder(token.account.collective) + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddons(token.account.collective) + resp <- Ok( + AddonList( + all.map(r => + Addon(r.id, r.name, r.version, r.description, r.originalUrl, r.created) + ) + ) + ) + } yield resp + + case req @ POST -> Root :? Sync(sync) => + def create(r: Option[RAddonArchive]) = + IdResult( + true, + r.fold("Addon submitted for installation")(r => + s"Addon installed: ${r.id.id}" + ), + r.map(_.id).getOrElse(Ident.unsafe("")) + ) + + for { + input <- req.as[AddonRegister] + install = backend.addons.registerAddon( + token.account.collective, + input.url, + None + ) + resp <- + if (sync) + install.flatMap( + _.fold(convertAddonValidationError[F], r => Ok(create(r._1.some))) + ) + else Background(wsTopic)(install).flatMap(_ => Ok(create(None))) + } yield resp + + case PUT -> Root / Ident(id) :? Sync(sync) => + def create(r: Option[AddonMeta]) = + BasicResult( + true, + r.fold("Addon updated in background")(m => + s"Addon updated: ${m.nameAndVersion}" + ) + ) + val update = backend.addons.refreshAddon(token.account.collective, id) + for { + resp <- + if (sync) + update.flatMap( + _.fold( + convertAddonValidationError[F], + r => Ok(create(r._2.some)) + ) + ) + else Background(wsTopic)(update).flatMap(_ => Ok(create(None))) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddon(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon deleted")) + else NotFound(BasicResult(false, "Addon not found")) + } yield resp + } + } + + def convertAddonValidationError[F[_]: Async]( + e: AddonValidationError + ): F[Response[F]] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + + def failWith(msg: String): F[Response[F]] = + Ok(IdResult(false, msg, Ident.unsafe(""))) + + e match { + case AddonValidationError.AddonNotFound => + NotFound(BasicResult(false, "Addon not found.")) + + case _ => + failWith(validationErrorToMessage(e)) + } + } + + object Sync extends FlagQueryParamMatcher("sync") +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala new file mode 100644 index 00000000..192f6ccb --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.effect.Async +import fs2.concurrent.Topic + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model.BasicResult +import docspell.restserver.Config +import docspell.restserver.http4s.Responses +import docspell.restserver.ws.OutputEvent + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.server.Router + +object AddonRoutes { + + def apply[F[_]: Async]( + cfg: Config, + wsTopic: Topic[F, OutputEvent], + backend: BackendApp[F], + token: AuthToken + ): HttpRoutes[F] = + if (cfg.backend.addons.enabled) + Router( + "archive" -> AddonArchiveRoutes(wsTopic, backend, token), + "run-config" -> AddonRunConfigRoutes(backend, token), + "run" -> AddonRunRoutes(backend, token) + ) + else + Responses.notFoundRoute(BasicResult(false, "Addons disabled")) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala new file mode 100644 index 00000000..d146eef1 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.OAddons +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunConfigRoutes { + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.addons.getAllAddonRunConfigs(token.account.collective) + resp <- Ok(AddonRunConfigList(all.map(convertInfoTask))) + } yield resp + + case req @ POST -> Root => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(Ident.unsafe(""), input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config added", id)) + ) + } yield resp + + case req @ PUT -> Root / Ident(id) => + for { + input <- req.as[AddonRunConfig] + data = convertInsertTask(id, input) + res <- data.flatTraverse(in => + backend.addons + .upsertAddonRunConfig(token.account.collective, in) + .map(_.leftMap(_.message)) + ) + resp <- res.fold( + msg => Ok(BasicResult(false, msg)), + id => Ok(IdResult(true, s"Addon run config updated", id)) + ) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + flag <- backend.addons.deleteAddonRunConfig(token.account.collective, id) + resp <- + if (flag) Ok(BasicResult(true, "Addon task deleted")) + else NotFound(BasicResult(false, "Addon task not found")) + } yield resp + } + } + + def convertInsertTask( + id: Ident, + t: AddonRunConfig + ): Either[String, OAddons.AddonRunInsert] = + for { + tr <- NonEmptyList + .fromList(t.trigger) + .toRight("At least one trigger is required") + ta <- NonEmptyList + .fromList(t.addons) + .toRight("At least one addon is required") + res = OAddons.AddonRunInsert( + id, + t.name, + t.enabled, + t.userId, + t.schedule, + tr, + ta.map(e => OAddons.AddonArgs(e.addonId, e.args)) + ) + } yield res + + def convertInfoTask(t: OAddons.AddonRunInfo): AddonRunConfig = + AddonRunConfig( + id = t.id, + name = t.name, + enabled = t.enabled, + userId = t.userId, + schedule = t.schedule, + trigger = t.triggered, + addons = t.addons.map { case (ra, raa) => + AddonRef(raa.addonId, ra.name, ra.version, ra.description, raa.args) + } + ) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala new file mode 100644 index 00000000..3b167b2d --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.syntax.all._ + +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model._ +import docspell.restserver.http4s.ThrowableResponseMapper + +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityCodec._ +import org.http4s.dsl.Http4sDsl + +object AddonRunRoutes { + + def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ThrowableResponseMapper {} + import dsl._ + + HttpRoutes.of { case req @ POST -> Root / "existingitem" => + for { + input <- req.as[AddonRunExistingItem] + _ <- backend.addons.runAddonForItem( + token.account, + NonEmptyList(input.itemId, input.additionalItems), + input.addonRunConfigIds.toSet + ) + resp <- Ok(BasicResult(true, "Job for running addons submitted.")) + } yield resp + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala index a713f160..1a8ebd6a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala @@ -117,7 +117,11 @@ object ItemMultiRoutes extends NonEmptyListSupport with MultiIdSupport { for { json <- req.as[ItemsAndRef] items <- requireNonEmpty(json.items) - res <- backend.item.setFolderMultiple(items, json.ref, user.account.collective) + res <- backend.item.setFolderMultiple( + items, + json.ref.map(_.id), + user.account.collective + ) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index d830fc6c..bcabb4c7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -218,7 +218,7 @@ object ItemRoutes { case req @ PUT -> Root / Ident(id) / "folder" => for { idref <- req.as[OptionalId] - res <- backend.item.setFolder(id, idref.id, user.account.collective) + res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective) resp <- Ok(Conversions.basicResult(res, "Folder updated")) } yield resp diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala index de54198c..5faf9f76 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala @@ -29,7 +29,8 @@ case class Flags( downloadAllMaxFiles: Int, downloadAllMaxSize: ByteSize, uiVersion: Int, - openIdAuth: List[Flags.OpenIdAuth] + openIdAuth: List[Flags.OpenIdAuth], + addonsEnabled: Boolean ) object Flags { @@ -47,7 +48,8 @@ object Flags { cfg.downloadAll.maxFiles, cfg.downloadAll.maxSize, uiVersion, - cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)) + cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)), + cfg.backend.addons.enabled ) final case class OpenIdAuth(provider: Ident, name: String) diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala new file mode 100644 index 00000000..b0870fc5 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +import cats.effect._ +import cats.syntax.all._ +import fs2.concurrent.Topic + +import docspell.logging.Logger + +/** Asynchronous operations that run on the rest-server can communicate their results via + * websocket. + */ +object Background { + // TODO avoid resubmitting same stuff + + def apply[F[_]: Async, A]( + wsTopic: Topic[F, OutputEvent], + logger: Option[Logger[F]] = None + )(run: F[A])(implicit enc: OutputEventEncoder[A]): F[Unit] = { + val log = logger.getOrElse(docspell.logging.getLogger[F]) + Async[F] + .background(run) + .use( + _.flatMap( + _.fold( + log.warn("The background operation has been cancelled!"), + ex => log.error(ex)("Error running background operation!"), + event => + event + .map(enc.encode) + .flatTap(ev => log.info(s"Sending response from async operation: $ev")) + .flatMap(wsTopic.publish1) + .void + ) + ) + ) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala index c7e57be4..5ac68c48 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala @@ -7,6 +7,7 @@ package docspell.restserver.ws import docspell.backend.auth.AuthToken +import docspell.backend.ops.AddonValidationError import docspell.common._ import io.circe._ @@ -55,6 +56,29 @@ object OutputEvent { Msg("jobs-waiting", count).asJson } + final case class AddonInstalled( + collective: Ident, + message: String, + error: Option[AddonValidationError], + addonId: Option[Ident], + originalUrl: Option[LenientUri] + ) extends OutputEvent { + def forCollective(token: AuthToken) = + token.account.collective == collective + + override def asJson = + Msg( + "addon-installed", + Map( + "success" -> error.isEmpty.asJson, + "error" -> error.asJson, + "addonId" -> addonId.asJson, + "addonUrl" -> originalUrl.asJson, + "message" -> message.asJson + ) + ).asJson + } + private case class Msg[A](tag: String, content: A) private object Msg { implicit def jsonEncoder[A: Encoder]: Encoder[Msg[A]] = diff --git a/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala new file mode 100644 index 00000000..4414dfad --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.ws + +trait OutputEventEncoder[A] { + def encode(a: A): OutputEvent +} + +object OutputEventEncoder { + def apply[A](implicit e: OutputEventEncoder[A]): OutputEventEncoder[A] = e + + def instance[A](f: A => OutputEvent): OutputEventEncoder[A] = + (a: A) => f(a) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala index 0594dcf8..4ca51dc9 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -6,6 +6,8 @@ package docspell.scheduler +import cats.effect.Sync + import docspell.common._ import docspell.logging.Logger @@ -25,4 +27,8 @@ trait Context[F[_], A] { self => def map[C](f: A => C): Context[F, C] + def unit: Context[F, Unit] = + map(_ => ()) + + def loadJob(implicit F: Sync[F]): F[Job[String]] } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala new file mode 100644 index 00000000..7a921769 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PermanentError.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +/** Special "marker" exception to indicate errors in tasks, that should NOT be retried. */ +final class PermanentError(cause: Throwable) extends RuntimeException(cause) { + override def fillInStackTrace() = this +} + +object PermanentError { + def apply(cause: Throwable): PermanentError = + new PermanentError(cause) + + def isPermanent(ex: Throwable): Boolean = + unapply(ex).isDefined + + def unapply(ex: Throwable): Option[Throwable] = + ex match { + case p: PermanentError => Some(p.getCause) + case _ => None + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala index d6868a08..b8ecd5fb 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala @@ -21,6 +21,11 @@ trait Task[F[_], A, B] { def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = Task(Task.toKleisli(this).andThen(f)) + def andThenC[C](f: (Context[F, A], B) => F[C])(implicit M: Monad[F]): Task[F, A, C] = { + val run = Task.toKleisli(this).run + Task(ctx => run(ctx).flatMap(b => f(ctx, b))) + } + def mapF[C](f: F[B] => F[C]): Task[F, A, C] = Task(Task.toKleisli(this).mapF(f)) diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala index 236c7ee6..bd3027fe 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala @@ -50,6 +50,9 @@ object UserTaskScope { def apply(collective: Ident): UserTaskScope = UserTaskScope.collective(collective) + def apply(collective: Ident, login: Option[Ident]): UserTaskScope = + login.map(AccountId(collective, _)).map(account).getOrElse(apply(collective)) + def system: UserTaskScope = collective(DocspellSystem.taskGroup) } diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala index 59016b9f..7b5a7615 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -24,6 +24,19 @@ class ContextImpl[F[_]: Functor, A]( val jobId: Ident ) extends Context[F, A] { + def loadJob(implicit F: Sync[F]): F[Job[String]] = + JobStoreImpl(store) + .findById(jobId) + .flatMap( + _.fold( + F.raiseError[Job[String]]( + new IllegalStateException(s"Job not found: ${jobId.id}") + ) + )( + F.pure + ) + ) + def setProgress(percent: Int): F[Unit] = { val pval = math.min(100, math.max(0, percent)) store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala index 5ebfd665..be263ad1 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala @@ -34,6 +34,7 @@ object LogSink { .capture("task", e.taskName) .capture("group", e.group) .capture("jobInfo", e.jobInfo) + .captureAll(e.data) e.level match { case LogLevel.Info => diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala index 50410e65..7e81cd1d 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala @@ -12,8 +12,7 @@ import cats.syntax.all._ import fs2.Stream import docspell.common.{Ident, LogLevel} -import docspell.logging -import docspell.logging.{Level, Logger} +import docspell.logging.{Level, LogEvent => DsLogEvent, Logger} /** Background tasks use this logger to emit the log events to a queue. The consumer is * [[LogSink]], which picks up log events in a separate thread. @@ -29,7 +28,7 @@ object QueueLogger { ): Logger[F] = new Logger[F] { - def log(logEvent: => logging.LogEvent) = + def log(logEvent: => DsLogEvent) = LogEvent .create[F]( jobId, diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index bc87d0fa..c114e2bc 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -267,27 +267,36 @@ final class SchedulerImpl[F[_]: Async]( .mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa) .mapF(_.attempt.flatMap { case Right(result) => - logger.info(s"Job execution successful: ${job.info}") - ctx.logger.info("Job execution successful") *> + logger.info(s"Job execution successful: ${job.info}") *> + ctx.logger.info("Job execution successful") *> (JobState.Success: JobState, result).pure[F] + + case Left(PermanentError(ex)) => + logger.warn(ex)("Task failed with permanent error") *> + ctx.logger + .warn(ex)("Task failed with permanent error!") + .as(JobState.failed -> JobTaskResult.empty) + case Left(ex) => state.get.map(_.wasCancelled(job)).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") - ctx.logger.error(ex)("Job execution failed (cancel = true)") *> + logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") *> + ctx.logger.error(ex)("Job execution failed (cancel = true)") *> (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] case false => QJob.exceedsRetries(job.id, config.retries, store).flatMap { case true => - logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) case false => - logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retrying later.") - .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) + logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") *> + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") + .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) } } }) diff --git a/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/h2/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql new file mode 100644 index 00000000..4bc97aba --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table `addon_archive`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `file_id` varchar(254) not null, + `original_url` varchar(2000), + `name` varchar(254) not null, + `version` varchar(254) not null, + `description` text, + `triggers` text not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`file_id`) references `filemeta`(`file_id`), + unique (`cid`, `original_url`), + unique (`cid`, `name`, `version`) +); + +create table `addon_run_config`( + `id` varchar(254) not null primary key, + `cid` varchar(254) not null, + `user_id` varchar(254), + `name` varchar(254) not null, + `enabled` boolean not null, + `created` timestamp not null, + foreign key (`cid`) references `collective`(`cid`), + foreign key (`user_id`) references `user_`(`uid`) +); + +create table `addon_run_config_addon` ( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `addon_id` varchar(254) not null, + `args` text not null, + `position` int not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + foreign key (`addon_id`) references `addon_archive`(`id`) on delete cascade +); + +create table `addon_run_config_trigger`( + `id` varchar(254) not null primary key, + `addon_run_config_id` varchar(254) not null, + `triggers` varchar(254) not null, + foreign key (`addon_run_config_id`) references `addon_run_config`(`id`) on delete cascade, + unique (`addon_run_config_id`, `triggers`) +); + +alter table `node` +add column (`server_secret` varchar(2000)); diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql new file mode 100644 index 00000000..0139c19d --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.36.0__addons.sql @@ -0,0 +1,47 @@ +create table "addon_archive"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "file_id" varchar(254) not null, + "original_url" varchar(2000), + "name" varchar(254) not null, + "version" varchar(254) not null, + "description" text, + "triggers" text not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("file_id") references "filemeta"("file_id"), + unique ("cid", "original_url"), + unique ("cid", "name", "version") +); + +create table "addon_run_config"( + "id" varchar(254) not null primary key, + "cid" varchar(254) not null, + "user_id" varchar(254), + "name" varchar(254) not null, + "enabled" boolean not null, + "created" timestamp not null, + foreign key ("cid") references "collective"("cid"), + foreign key ("user_id") references "user_"("uid") +); + +create table "addon_run_config_addon" ( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "addon_id" varchar(254) not null, + "args" text not null, + "position" int not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + foreign key ("addon_id") references "addon_archive"("id") on delete cascade +); + +create table "addon_run_config_trigger"( + "id" varchar(254) not null primary key, + "addon_run_config_id" varchar(254) not null, + "triggers" varchar(254) not null, + foreign key ("addon_run_config_id") references "addon_run_config"("id") on delete cascade, + unique ("addon_run_config_id", "triggers") +); + +alter table "node" +add column "server_secret" varchar; diff --git a/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala new file mode 100644 index 00000000..df69f421 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/file/FileUrlReader.scala @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.file + +import cats.data.{NonEmptyList => Nel} +import cats.effect.Sync +import cats.syntax.all._ +import fs2.Stream + +import docspell.common.{FileKey, LenientUri, UrlReader} + +import binny.BinaryId + +object FileUrlReader { + + private val scheme: String = "docspell-file" + + def url(key: FileKey): LenientUri = + LenientUri( + scheme = Nel.of(scheme), + authority = Some(""), + path = LenientUri.NonEmptyPath( + Nel.of(key.collective.id, key.category.id.id, key.id.id) + ), + query = None, + fragment = None + ) + + def apply[F[_]: Sync](repo: FileRepository[F]): UrlReader[F] = + UrlReader.instance { url => + url.scheme.head match { + case `scheme` => + Stream + .emit(urlToFileKey(url)) + .covary[F] + .rethrow + .evalMap(key => repo.findMeta(key).map(m => (key, m))) + .flatMap { + case _ -> Some(m) => repo.getBytes(m.id) + case key -> None => + Stream.raiseError( + new NoSuchElementException( + s"File not found for url '${url.asString}' (key=$key)" + ) + ) + } + + case _ => + UrlReader.defaultReader[F].apply(url) + } + } + + private[file] def urlToFileKey(url: LenientUri): Either[Throwable, FileKey] = + BinnyUtils + .binaryIdToFileKey(BinaryId(url.host match { + case Some(h) if h.nonEmpty => s"$h${url.path.asString}" + case _ => url.path.segments.mkString("/") + })) + .leftMap(new IllegalArgumentException(_)) +} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala index 738a078b..6e919330 100644 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala @@ -9,6 +9,7 @@ package docspell.store.impl import java.time.format.DateTimeFormatter import java.time.{Instant, LocalDate} +import docspell.addons.AddonTriggerType import docspell.common._ import docspell.common.syntax.all._ import docspell.jsonminiq.JsonMiniQuery @@ -31,9 +32,9 @@ trait DoobieMeta extends EmilDoobieMeta { implicit val sqlLogging: LogHandler = LogHandler { case e @ Success(_, _, _, _) => - DoobieMeta.logger.trace("SQL " + e) + DoobieMeta.logger.trace(s"SQL: $e") case e => - DoobieMeta.logger.error(s"SQL Failure: $e") + DoobieMeta.logger.warn(s"SQL Failure: $e") } def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = @@ -41,6 +42,12 @@ trait DoobieMeta extends EmilDoobieMeta { e.apply(a).noSpaces ) + implicit val metaAddonTriggerType: Meta[AddonTriggerType] = + Meta[String].timap(AddonTriggerType.unsafeFromString)(_.name) + + implicit val metaAddonTriggerTypeSet: Meta[Set[AddonTriggerType]] = + jsonMeta[Set[AddonTriggerType]] + implicit val metaBinaryId: Meta[BinaryId] = Meta[String].timap(BinaryId.apply)(_.id) diff --git a/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala new file mode 100644 index 00000000..94e51f96 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/AttachedFile.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.queries + +import docspell.common.BaseJsonCodecs._ +import docspell.common._ + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} +import scodec.bits.ByteVector + +/** Information about an attachment file (can be attachment-source or attachment) */ +case class AttachedFile( + id: Ident, + name: Option[String], + position: Int, + language: Option[Language], + mimetype: MimeType, + length: ByteSize, + checksum: ByteVector +) + +object AttachedFile { + + implicit val jsonDecoder: Decoder[AttachedFile] = deriveDecoder + implicit val jsonEncoder: Encoder[AttachedFile] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 5cdcafb3..6e90cc86 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -22,10 +22,37 @@ import doobie._ object QAttachment { private val a = RAttachment.as("a") + private val as = RAttachmentSource.as("ats") private val item = RItem.as("i") private val am = RAttachmentMeta.as("am") private val c = RCollective.as("c") private val im = RItemProposal.as("im") + private val fm = RFileMeta.as("fm") + + def attachmentSourceFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(as.id, as.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(as, a.id === as.id) + .innerJoin(fm, fm.id === as.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] + + def attachmentFile(itemId: Ident): ConnectionIO[List[AttachedFile]] = + Select( + combineNel( + select(a.id, a.name, a.position, am.language), + select(fm.mimetype, fm.length, fm.checksum) + ), + from(a) + .innerJoin(fm, fm.id === a.fileId) + .leftJoin(am, am.id === a.id), + a.itemId === itemId + ).orderBy(a.position).build.query[AttachedFile].to[List] def deletePreview[F[_]: Sync](store: Store[F])(attachId: Ident): F[Int] = { val findPreview = @@ -163,6 +190,17 @@ object QAttachment { q.query[RAttachmentMeta].option } + def getAttachmentMetaOfItem(itemId: Ident): ConnectionIO[Vector[RAttachmentMeta]] = + Select( + select(am.all), + from(am) + .innerJoin(a, a.id === am.id), + a.itemId === itemId + ).orderBy(a.position.asc) + .build + .query[RAttachmentMeta] + .to[Vector] + case class ContentAndName( id: Ident, item: Ident, @@ -175,6 +213,7 @@ object QAttachment { def allAttachmentMetaAndName( coll: Option[Ident], itemIds: Option[Nel[Ident]], + itemStates: Nel[ItemState], chunkSize: Int ): Stream[ConnectionIO, ContentAndName] = Select( @@ -192,7 +231,7 @@ object QAttachment { .innerJoin(item, item.id === a.itemId) .innerJoin(c, c.id === item.cid) ).where( - item.state.in(ItemState.validStates) &&? + item.state.in(itemStates) &&? itemIds.map(ids => item.id.in(ids)) &&? coll.map(cid => item.cid === cid) ).build diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala new file mode 100644 index 00000000..69c6826a --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigData.scala @@ -0,0 +1,155 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList +import cats.syntax.all._ +import fs2.Stream + +import docspell.addons.AddonTriggerType +import docspell.common.{Ident, Timestamp} +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ + +case class AddonRunConfigData( + runConfig: RAddonRunConfig, + addons: List[RAddonRunConfigAddon], + triggers: List[RAddonRunConfigTrigger] +) + +object AddonRunConfigData { + + def findAll( + cid: Ident, + enabled: Option[Boolean] = None, + trigger: Set[AddonTriggerType] = Set.empty, + configIds: Set[Ident] = Set.empty + ): ConnectionIO[List[AddonRunConfigData]] = + for { + runConfigs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + addons <- runConfigs.traverse(t => + RAddonRunConfigAddon.findByRunConfig(t.id).map(as => t.id -> as) + ) + addonMap = addons.toMap + triggers <- runConfigs.traverse(t => + RAddonRunConfigTrigger.findByRunConfig(t.id).map(ts => t.id -> ts) + ) + triggerMap = triggers.toMap + result = runConfigs.map(t => + AddonRunConfigData(t, addonMap(t.id), triggerMap(t.id)) + ) + } yield result + + /** Inserts new, creating new identifiers */ + def insert(task: AddonRunConfigData): ConnectionIO[Ident] = + for { + tid <- Ident.randomId[ConnectionIO] + now <- Timestamp.current[ConnectionIO] + tr = task.runConfig.copy(id = tid, created = now) + _ <- RAddonRunConfig.insert(tr) + _ <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigTrigger.insert) + } + _ <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = tid)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield tid + + /** Updates the task, keeping its id but replacing all related objects */ + def update(task: AddonRunConfigData): ConnectionIO[Int] = + for { + n1 <- RAddonRunConfig.update(task.runConfig) + _ <- RAddonRunConfigTrigger.deleteAllForConfig(task.runConfig.id) + _ <- RAddonRunConfigAddon.deleteAllForConfig(task.runConfig.id) + tts <- task.triggers.traverse { t => + Ident + .randomId[ConnectionIO] + .map(id => t.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigTrigger.insert) + } + tas <- task.addons.traverse { a => + Ident + .randomId[ConnectionIO] + .map(id => a.copy(id = id, runConfigId = task.runConfig.id)) + .flatMap(RAddonRunConfigAddon.insert) + } + } yield n1 + tts.sum + tas.sum + + def findEnabledRef( + cid: Ident, + taskId: Ident + ): ConnectionIO[List[(RAddonArchive, RAddonRunConfigAddon)]] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + + Select( + combineNel(select(aa.all), select(ta.all)), + from(run) + .innerJoin(ta, ta.runConfigId === run.id) + .innerJoin(aa, aa.id === ta.addonId), + run.cid === cid && run.enabled === true && run.id === taskId + ).orderBy(ta.position.asc) + .build + .query[(RAddonArchive, RAddonRunConfigAddon)] + .to[List] + } + + def findEnabledRefs( + cid: Ident, + trigger: AddonTriggerType, + addonTaskIds: Set[Ident] + ): Stream[ConnectionIO, (RAddonRunConfig, List[(RAddonArchive, String)])] = { + val run = RAddonRunConfig.as("run") + val aa = RAddonArchive.as("aa") + val ta = RAddonRunConfigAddon.as("ta") + val tt = RAddonRunConfigTrigger.as("tt") + + val taskIdFilter = NonEmptyList + .fromList(addonTaskIds.toList) + .map(nel => run.id.in(nel)) + val validTasks = TableDef("valid_task") + val validTaskId = Column[Ident]("id", validTasks) + val query = + withCte( + validTasks -> Select( + select(run.all), + from(run) + .innerJoin(tt, tt.runConfigId === run.id), + run.cid === cid && run.enabled === true && tt.trigger === trigger &&? taskIdFilter + ).distinct + )( + Select( + combineNel( + select(run.all.map(_.copy(table = validTasks))), + select(aa.all), + select(ta.args) + ), + from(validTasks) + .innerJoin(ta, ta.runConfigId === validTaskId) + .innerJoin(aa, aa.id === ta.addonId) + ).orderBy(validTaskId) + ).build + + query + .query[(RAddonRunConfig, RAddonArchive, String)] + .stream + .groupAdjacentBy(_._1.id) + .map { case (_, chunk) => + val list = chunk.toList + (list.head._1, list.map(e => (e._2, e._3))) + } + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala new file mode 100644 index 00000000..94ddb6bc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/AddonRunConfigResolved.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.OptionT +import cats.syntax.all._ + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ +import docspell.store.records.AddonRunConfigResolved.AddonRef + +import doobie._ +import doobie.implicits._ + +final case class AddonRunConfigResolved( + config: RAddonRunConfig, + refs: List[AddonRef], + trigger: List[RAddonRunConfigTrigger] +) {} + +object AddonRunConfigResolved { + + case class AddonRef(archive: RAddonArchive, ref: RAddonRunConfigAddon) + + def findAddonRefs(configId: Ident): ConnectionIO[List[AddonRef]] = { + val ca = RAddonRunConfigAddon.as("ca") + val aa = RAddonArchive.as("aa") + Select( + select(combineNel(aa.all, ca.all)), + from(ca) + .innerJoin(aa, aa.id === ca.addonId), + ca.runConfigId === configId + ).build.query[AddonRef].to[List] + } + + def getRefsAndTrigger( + configId: Ident + ): ConnectionIO[(List[AddonRef], List[RAddonRunConfigTrigger])] = + (findAddonRefs(configId), RAddonRunConfigTrigger.findByRunConfig(configId)).tupled + + def findById( + configId: Ident, + collective: Ident, + enabled: Option[Boolean] + ): ConnectionIO[Option[AddonRunConfigResolved]] = + (for { + cfg <- OptionT(RAddonRunConfig.findById(collective, configId)) + .filter(c => enabled.isEmpty || enabled == c.enabled.some) + (refs, tri) <- OptionT.liftF(getRefsAndTrigger(configId)) + } yield AddonRunConfigResolved(cfg, refs, tri)).value + + def findAllForCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[AddonRunConfigResolved]] = + for { + cfgs <- RAddonRunConfig.findByCollective(cid, enabled, trigger, configIds) + result <- cfgs.traverse(ac => + getRefsAndTrigger(ac.id).map { case (refs, tri) => + AddonRunConfigResolved(ac, refs, tri) + } + ) + } yield result +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala new file mode 100644 index 00000000..05e2fc15 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonArchive.scala @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.{AddonArchive, AddonMeta, AddonTriggerType} +import docspell.common._ +import docspell.store.file.FileUrlReader +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class RAddonArchive( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + name: String, + version: String, + description: Option[String], + triggers: Set[AddonTriggerType], + created: Timestamp +) { + + def nameAndVersion: String = + s"${name}-${version}" + + def isUnchanged(meta: AddonMeta): Boolean = + name == meta.meta.name && + version == meta.meta.version && + description == meta.meta.description + + def isChanged(meta: AddonMeta): Boolean = + !isUnchanged(meta) + + def asArchive: AddonArchive = + AddonArchive(FileUrlReader.url(fileId), name, version) + + def update(file: FileKey, meta: AddonMeta): RAddonArchive = + copy( + fileId = file, + name = meta.meta.name, + version = meta.meta.version, + description = meta.meta.description, + triggers = meta.triggers.getOrElse(Set.empty) + ) +} + +object RAddonArchive { + case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_archive" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val fileId = Column[FileKey]("file_id", this) + val originalUrl = Column[LenientUri]("original_url", this) + val name = Column[String]("name", this) + val version = Column[String]("version", this) + val description = Column[String]("description", this) + val triggers = Column[Set[AddonTriggerType]]("triggers", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of( + id, + cid, + fileId, + originalUrl, + name, + version, + description, + triggers, + created + ) + } + + def apply( + id: Ident, + cid: Ident, + fileId: FileKey, + originalUrl: Option[LenientUri], + meta: AddonMeta, + created: Timestamp + ): RAddonArchive = + RAddonArchive( + id, + cid, + fileId, + originalUrl, + meta.meta.name, + meta.meta.version, + meta.meta.description, + meta.triggers.getOrElse(Set.empty), + created + ) + + def as(alias: String): Table = + Table(Some(alias)) + + val T = Table(None) + + def insert(r: RAddonArchive, silent: Boolean): ConnectionIO[Int] = { + val values = + sql"${r.id}, ${r.cid}, ${r.fileId}, ${r.originalUrl}, ${r.name}, ${r.version}, ${r.description}, ${r.triggers}, ${r.created}" + + if (silent) DML.insertSilent(T, T.all, values) + else DML.insert(T, T.all, values) + } + + def existsByUrl(cid: Ident, url: LenientUri): ConnectionIO[Boolean] = + Select( + select(count(T.id)), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[Int].unique.map(_ > 0) + + def findByUrl(cid: Ident, url: LenientUri): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.originalUrl === url + ).build.query[RAddonArchive].option + + def findByNameAndVersion( + cid: Ident, + name: String, + version: String + ): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.name === name && T.version === version + ).build.query[RAddonArchive].option + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id === id + ).build.query[RAddonArchive].option + + def findByIds(cid: Ident, ids: NonEmptyList[Ident]): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid && T.id.in(ids) + ).orderBy(T.name).build.query[RAddonArchive].to[List] + + def update(r: RAddonArchive): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id && T.cid === r.cid, + DML.set( + T.fileId.setTo(r.fileId), + T.originalUrl.setTo(r.originalUrl), + T.name.setTo(r.name), + T.version.setTo(r.version), + T.description.setTo(r.description), + T.triggers.setTo(r.triggers) + ) + ) + + def listAll(cid: Ident): ConnectionIO[List[RAddonArchive]] = + Select( + select(T.all), + from(T), + T.cid === cid + ).orderBy(T.name.asc).build.query[RAddonArchive].to[List] + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) + + implicit val jsonDecoder: Decoder[RAddonArchive] = deriveDecoder + implicit val jsonEncoder: Encoder[RAddonArchive] = deriveEncoder +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala new file mode 100644 index 00000000..70460aaa --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfig.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfig( + id: Ident, + cid: Ident, + userId: Option[Ident], + name: String, + enabled: Boolean, + created: Timestamp +) + +object RAddonRunConfig { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config" + + val id = Column[Ident]("id", this) + val cid = Column[Ident]("cid", this) + val userId = Column[Ident]("user_id", this) + val name = Column[String]("name", this) + val enabled = Column[Boolean]("enabled", this) + val created = Column[Timestamp]("created", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, cid, userId, name, enabled, created) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfig): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.cid}, ${r.userId}, ${r.name}, ${r.enabled}, ${r.created}" + ) + + def update(r: RAddonRunConfig): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.name.setTo(r.name), + T.enabled.setTo(r.enabled), + T.userId.setTo(r.userId) + ) + ) + + def findById(cid: Ident, id: Ident): ConnectionIO[Option[RAddonRunConfig]] = + Select(select(T.all), from(T), T.cid === cid && T.id === id).build + .query[RAddonRunConfig] + .option + + def findByCollective( + cid: Ident, + enabled: Option[Boolean], + trigger: Set[AddonTriggerType], + configIds: Set[Ident] + ): ConnectionIO[List[RAddonRunConfig]] = { + val ac = RAddonRunConfig.as("ac") + val tt = RAddonRunConfigTrigger.as("tt") + val filter = + ac.cid === cid &&? + enabled.map(e => ac.enabled === e) &&? + NonEmptyList.fromList(configIds.toList).map(ids => ac.id.in(ids)) + + val selectConfigs = + NonEmptyList.fromList(trigger.toList) match { + case Some(tri) => + Select( + select(ac.all), + from(ac).innerJoin(tt, tt.runConfigId === ac.id), + filter && tt.trigger.in(tri) + ) + case None => + Select(select(ac.all), from(ac), filter) + } + + selectConfigs.build.query[RAddonRunConfig].to[List] + } + + def deleteById(cid: Ident, id: Ident): ConnectionIO[Int] = + DML.delete(T, T.cid === cid && T.id === id) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala new file mode 100644 index 00000000..aa95ca35 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigAddon.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigAddon( + id: Ident, + runConfigId: Ident, + addonId: Ident, + args: String, + position: Int +) + +object RAddonRunConfigAddon { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_addon" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val addonId = Column[Ident]("addon_id", this) + val args = Column[String]("args", this) + val position = Column[Int]("position", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, addonId, args, position) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def insert(r: RAddonRunConfigAddon): ConnectionIO[Int] = + DML.insert( + T, + T.all, + sql"${r.id}, ${r.runConfigId}, ${r.addonId}, ${r.args}, ${r.position}" + ) + + def updateArgs(addonTaskId: Ident, addonId: Ident, args: String): ConnectionIO[Int] = + DML.update( + T, + T.runConfigId === addonTaskId && T.addonId === addonId, + DML.set( + T.args.setTo(args) + ) + ) + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigAddon]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId) + .orderBy(T.position.asc) + .build + .query[RAddonRunConfigAddon] + .to[List] + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala new file mode 100644 index 00000000..fccd81f2 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAddonRunConfigTrigger.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.records + +import cats.data.NonEmptyList + +import docspell.addons.AddonTriggerType +import docspell.common._ +import docspell.store.qb.DSL._ +import docspell.store.qb._ + +import doobie._ +import doobie.implicits._ + +final case class RAddonRunConfigTrigger( + id: Ident, + runConfigId: Ident, + trigger: AddonTriggerType +) + +object RAddonRunConfigTrigger { + final case class Table(alias: Option[String]) extends TableDef { + val tableName = "addon_run_config_trigger" + + val id = Column[Ident]("id", this) + val runConfigId = Column[Ident]("addon_run_config_id", this) + val trigger = Column[AddonTriggerType]("triggers", this) + + val all: NonEmptyList[Column[_]] = + NonEmptyList.of(id, runConfigId, trigger) + } + + def as(alias: String): Table = Table(Some(alias)) + val T = Table(None) + + def deleteAllForConfig(addonTaskId: Ident): ConnectionIO[Int] = + DML.delete(T, T.runConfigId === addonTaskId) + + def insert(r: RAddonRunConfigTrigger): ConnectionIO[Int] = + DML.insert(T, T.all, sql"${r.id}, ${r.runConfigId}, ${r.trigger}") + + def insertAll( + addonTaskId: Ident, + triggers: NonEmptyList[AddonTriggerType] + ): ConnectionIO[Int] = { + val records = triggers.traverse(t => + Ident.randomId[ConnectionIO].map(id => RAddonRunConfigTrigger(id, addonTaskId, t)) + ) + val inserts = + s"INSERT INTO ${T.tableName} (id, addon_run_config_id, trigger) VALUES (?,?,?)" + records.flatMap(rs => Update[RAddonRunConfigTrigger](inserts).updateMany(rs)) + } + + def findByRunConfig(addonTaskId: Ident): ConnectionIO[List[RAddonRunConfigTrigger]] = + Select(select(T.all), from(T), T.runConfigId === addonTaskId).build + .query[RAddonRunConfigTrigger] + .to[List] +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 64599d6e..cb53f3d8 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -119,6 +119,9 @@ object RAttachmentMeta { def updatePageCount(mid: Ident, pageCount: Option[Int]): ConnectionIO[Int] = DML.update(T, T.id === mid, DML.set(T.pages.setTo(pageCount))) + def updateContent(id: Ident, text: String): ConnectionIO[Int] = + DML.update(T, T.id === id, DML.set(T.content.setTo(text))) + def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala index 3b9d23aa..6ca4bc8e 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentPreview.scala @@ -6,7 +6,8 @@ package docspell.store.records -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, OptionT} +import cats.syntax.all._ import docspell.common.{FileKey, _} import docspell.store.qb.DSL._ @@ -44,12 +45,29 @@ object RAttachmentPreview { def insert(v: RAttachmentPreview): ConnectionIO[Int] = DML.insert(T, T.all, fr"${v.id},${v.fileId},${v.name},${v.created}") + def update(r: RAttachmentPreview): ConnectionIO[Int] = + DML.update( + T, + T.id === r.id, + DML.set( + T.fileId.setTo(r.fileId), + T.name.setTo(r.name) + ) + ) + def findById(attachId: Ident): ConnectionIO[Option[RAttachmentPreview]] = run(select(T.all), from(T), T.id === attachId).query[RAttachmentPreview].option def delete(attachId: Ident): ConnectionIO[Int] = DML.delete(T, T.id === attachId) + def upsert(r: RAttachmentPreview): ConnectionIO[Option[FileKey]] = + OptionT(findById(r.id)) + .semiflatMap(existing => + update(existing.copy(fileId = r.fileId, name = r.name)).as(Some(existing.fileId)) + ) + .getOrElseF(insert(r).as(None)) + def findByIdAndCollective( attachId: Ident, collective: Ident diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 17faebba..2ac753ca 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -309,20 +309,22 @@ object RItem { def updateFolder( itemId: Ident, coll: Ident, - folderId: Option[Ident] - ): ConnectionIO[Int] = + folderIdOrName: Option[String] + ): ConnectionIO[(Int, Option[Ident])] = for { t <- currentTime - fid <- folderId match { - case Some(f) => RFolder.requireIdByIdOrName(f, f.id, coll).map(_.some) - case None => None.pure[ConnectionIO] + fid <- folderIdOrName match { + case Some(f) => + val fid = Ident.fromString(f).getOrElse(Ident.unsafe("")) + RFolder.requireIdByIdOrName(fid, f, coll).map(_.some) + case None => None.pure[ConnectionIO] } n <- DML.update( T, T.cid === coll && T.id === itemId, DML.set(T.folder.setTo(fid), T.updated.setTo(t)) ) - } yield n + } yield (n, fid) def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = for { @@ -334,6 +336,26 @@ object RItem { ) } yield n + def appendNotes( + itemId: Ident, + cid: Ident, + text: String, + sep: Option[String] + ): ConnectionIO[Option[String]] = { + val curNotes = + Select(select(T.notes), from(T), T.cid === cid && T.id === itemId).build + .query[Option[String]] + .option + + curNotes.flatMap { + case Some(notes) => + val newText = notes.map(_ + sep.getOrElse("")).getOrElse("") + text + updateNotes(itemId, cid, Some(newText)).as(newText.some) + case None => + (None: Option[String]).pure[ConnectionIO] + } + } + def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = for { t <- currentTime diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index fdcb8dd4..ce3c1daa 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -16,6 +16,7 @@ import docspell.store.qb._ import doobie._ import doobie.implicits._ +import scodec.bits.ByteVector case class RNode( id: Ident, @@ -23,13 +24,19 @@ case class RNode( url: LenientUri, updated: Timestamp, created: Timestamp, - notFound: Int + notFound: Int, + serverSecret: Option[ByteVector] ) {} object RNode { - def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] = - Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0)) + def apply[F[_]: Sync]( + id: Ident, + nodeType: NodeType, + uri: LenientUri, + serverSecret: Option[ByteVector] + ): F[RNode] = + Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0, serverSecret)) final case class Table(alias: Option[String]) extends TableDef { val tableName = "node" @@ -40,7 +47,9 @@ object RNode { val updated = Column[Timestamp]("updated", this) val created = Column[Timestamp]("created", this) val notFound = Column[Int]("not_found", this) - val all = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created, notFound) + val serverSecret = Column[ByteVector]("server_secret", this) + val all = NonEmptyList + .of[Column[_]](id, nodeType, url, updated, created, notFound, serverSecret) } def as(alias: String): Table = @@ -52,7 +61,7 @@ object RNode { DML.insert( t, t.all, - fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound}" + fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound},${v.serverSecret}" ) } @@ -65,6 +74,7 @@ object RNode { DML.set( t.nodeType.setTo(v.nodeType), t.url.setTo(v.url), + t.serverSecret.setTo(v.serverSecret), t.updated.setTo(v.updated) ) ) diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index c100f19d..651c99fe 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -152,6 +152,14 @@ object RUser { .query[Ident] .option + case class IdAndLogin(uid: Ident, login: Ident) + def getIdByIdOrLogin(idOrLogin: Ident): ConnectionIO[Option[IdAndLogin]] = + Select( + select(T.uid, T.login), + from(T), + T.uid === idOrLogin || T.login === idOrLogin + ).build.query[IdAndLogin].option + def getIdByAccount(account: AccountId): ConnectionIO[Ident] = OptionT(findIdByAccount(account)).getOrElseF( Sync[ConnectionIO].raiseError( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cd832f82..372e708e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -264,6 +264,9 @@ object Dependencies { val circeGenericExtra = Seq( "io.circe" %% "circe-generic-extras" % CirceVersion ) + val circeYaml = Seq( + "io.circe" %% "circe-yaml" % CirceVersion + ) // // https://github.com/Log4s/log4s;ASL 2.0 // val loggingApi = Seq( From 73747c4ea3dbd786dcedd0fcfd9f9252df7a7783 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 8 May 2022 14:01:41 +0200 Subject: [PATCH 2/2] Basic ui for addons --- modules/webapp/src/main/elm/Api.elm | 106 +++ modules/webapp/src/main/elm/App/Update.elm | 6 +- .../src/main/elm/Comp/AddonArchiveForm.elm | 106 +++ .../src/main/elm/Comp/AddonArchiveManage.elm | 429 +++++++++++ .../src/main/elm/Comp/AddonArchiveTable.elm | 72 ++ .../src/main/elm/Comp/AddonRunConfigForm.elm | 709 ++++++++++++++++++ .../main/elm/Comp/AddonRunConfigManage.elm | 364 +++++++++ .../src/main/elm/Comp/AddonRunConfigTable.elm | 79 ++ .../src/main/elm/Comp/ItemDetail/Model.elm | 16 + .../main/elm/Comp/ItemDetail/RunAddonForm.elm | 78 ++ .../src/main/elm/Comp/ItemDetail/Update.elm | 74 ++ .../src/main/elm/Comp/ItemDetail/View2.elm | 33 + .../webapp/src/main/elm/Data/AddonTrigger.elm | 59 ++ modules/webapp/src/main/elm/Data/CalEvent.elm | 8 +- modules/webapp/src/main/elm/Data/Flags.elm | 1 + modules/webapp/src/main/elm/Data/Icons.elm | 22 +- .../webapp/src/main/elm/Data/ServerEvent.elm | 25 +- .../elm/Messages/Comp/AddonArchiveForm.elm | 54 ++ .../elm/Messages/Comp/AddonArchiveManage.elm | 86 +++ .../elm/Messages/Comp/AddonArchiveTable.elm | 42 ++ .../elm/Messages/Comp/AddonRunConfigForm.elm | 108 +++ .../Messages/Comp/AddonRunConfigManage.elm | 79 ++ .../elm/Messages/Comp/AddonRunConfigTable.elm | 50 ++ .../src/main/elm/Messages/Comp/ItemDetail.elm | 17 + .../Messages/Comp/ItemDetail/RunAddonForm.elm | 49 ++ .../src/main/elm/Messages/Page/ManageData.elm | 18 + .../src/main/elm/Page/ManageData/Data.elm | 18 + .../src/main/elm/Page/ManageData/Update.elm | 49 +- .../src/main/elm/Page/ManageData/View2.elm | 74 +- modules/webapp/src/main/elm/Styles.elm | 7 +- modules/webapp/src/main/elm/Util/List.elm | 30 + modules/webapp/src/main/elm/Util/String.elm | 22 +- .../src/main/styles/custom-components.css | 4 + 33 files changed, 2881 insertions(+), 13 deletions(-) create mode 100644 modules/webapp/src/main/elm/Comp/AddonArchiveForm.elm create mode 100644 modules/webapp/src/main/elm/Comp/AddonArchiveManage.elm create mode 100644 modules/webapp/src/main/elm/Comp/AddonArchiveTable.elm create mode 100644 modules/webapp/src/main/elm/Comp/AddonRunConfigForm.elm create mode 100644 modules/webapp/src/main/elm/Comp/AddonRunConfigManage.elm create mode 100644 modules/webapp/src/main/elm/Comp/AddonRunConfigTable.elm create mode 100644 modules/webapp/src/main/elm/Comp/ItemDetail/RunAddonForm.elm create mode 100644 modules/webapp/src/main/elm/Data/AddonTrigger.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonArchiveForm.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonArchiveManage.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonArchiveTable.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigForm.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigManage.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigTable.elm create mode 100644 modules/webapp/src/main/elm/Messages/Comp/ItemDetail/RunAddonForm.elm diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 919f8a1b..ff22e2a6 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -18,6 +18,14 @@ module Api exposing , addShare , addTag , addTagsMultiple + , addonRunConfigDelete + , addonRunConfigGet + , addonRunConfigSet + , addonRunExistingItem + , addonsDelete + , addonsGetAll + , addonsInstall + , addonsUpdate , attachmentPreviewURL , bookmarkNameExists , cancelJob @@ -211,6 +219,11 @@ module Api exposing , versionInfo ) +import Api.Model.AddonList exposing (AddonList) +import Api.Model.AddonRegister exposing (AddonRegister) +import Api.Model.AddonRunConfig exposing (AddonRunConfig) +import Api.Model.AddonRunConfigList exposing (AddonRunConfigList) +import Api.Model.AddonRunExistingItem exposing (AddonRunExistingItem) import Api.Model.AttachmentMeta exposing (AttachmentMeta) import Api.Model.AuthResult exposing (AuthResult) import Api.Model.BasicResult exposing (BasicResult) @@ -3156,6 +3169,99 @@ shareDownloadAllLink flags id = +--- Addons + + +addonsGetAll : Flags -> (Result Http.Error AddonList -> msg) -> Cmd msg +addonsGetAll flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/archive" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.AddonList.decoder + } + + +addonsDelete : Flags -> String -> (Result Http.Error BasicResult -> msg) -> Cmd msg +addonsDelete flags addonId receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/archive/" ++ addonId + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +addonsInstall : Flags -> AddonRegister -> (Result Http.Error BasicResult -> msg) -> Cmd msg +addonsInstall flags addon receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/archive" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.AddonRegister.encode addon) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +addonsUpdate : Flags -> String -> (Result Http.Error BasicResult -> msg) -> Cmd msg +addonsUpdate flags addonId receive = + Http2.authPut + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/archive/" ++ addonId + , account = getAccount flags + , body = Http.emptyBody + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +addonRunConfigGet : Flags -> (Result Http.Error AddonRunConfigList -> msg) -> Cmd msg +addonRunConfigGet flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/run-config" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.AddonRunConfigList.decoder + } + + +addonRunConfigSet : + Flags + -> AddonRunConfig + -> (Result Http.Error BasicResult -> msg) + -> Cmd msg +addonRunConfigSet flags cfg receive = + if cfg.id == "" then + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/run-config" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.AddonRunConfig.encode cfg) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + else + Http2.authPut + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/run-config/" ++ cfg.id + , account = getAccount flags + , body = Http.jsonBody (Api.Model.AddonRunConfig.encode cfg) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +addonRunConfigDelete : Flags -> String -> (Result Http.Error BasicResult -> msg) -> Cmd msg +addonRunConfigDelete flags id receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/run-config/" ++ id + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +addonRunExistingItem : Flags -> AddonRunExistingItem -> (Result Http.Error BasicResult -> msg) -> Cmd msg +addonRunExistingItem flags input receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/addon/run/existingitem" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.AddonRunExistingItem.encode input) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + + --- Helper diff --git a/modules/webapp/src/main/elm/App/Update.elm b/modules/webapp/src/main/elm/App/Update.elm index 7f98c14c..67b9f416 100644 --- a/modules/webapp/src/main/elm/App/Update.elm +++ b/modules/webapp/src/main/elm/App/Update.elm @@ -14,6 +14,7 @@ import Api import App.Data exposing (..) import Browser exposing (UrlRequest(..)) import Browser.Navigation as Nav +import Comp.AddonArchiveManage import Comp.DownloadAll import Data.AppEvent exposing (AppEvent(..)) import Data.Environment as Env @@ -345,6 +346,9 @@ updateWithSub msg model = Ok (JobsWaiting n) -> ( { model | jobsWaiting = max 0 n }, Cmd.none, Sub.none ) + Ok (AddonInstalled info) -> + updateManageData (Page.ManageData.Data.AddonArchiveMsg <| Comp.AddonArchiveManage.addonInstallResult info) model + Err _ -> ( model, Cmd.none, Sub.none ) @@ -640,7 +644,7 @@ updateManageData : Page.ManageData.Data.Msg -> Model -> ( Model, Cmd Msg, Sub Ms updateManageData lmsg model = let ( lm, lc, ls ) = - Page.ManageData.Update.update model.flags lmsg model.manageDataModel + Page.ManageData.Update.update model.flags model.uiSettings lmsg model.manageDataModel in ( { model | manageDataModel = lm } , Cmd.map ManageDataMsg lc diff --git a/modules/webapp/src/main/elm/Comp/AddonArchiveForm.elm b/modules/webapp/src/main/elm/Comp/AddonArchiveForm.elm new file mode 100644 index 00000000..5030eef5 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonArchiveForm.elm @@ -0,0 +1,106 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonArchiveForm exposing (Model, Msg, get, init, initWith, update, view) + +import Api.Model.Addon exposing (Addon) +import Comp.Basic as B +import Data.Flags exposing (Flags) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Messages.Comp.AddonArchiveForm exposing (Texts) +import Styles as S +import Util.Maybe + + +type alias Model = + { addon : Addon + , url : Maybe String + } + + +init : ( Model, Cmd Msg ) +init = + ( { addon = Api.Model.Addon.empty + , url = Nothing + } + , Cmd.none + ) + + +initWith : Addon -> ( Model, Cmd Msg ) +initWith a = + ( { addon = a + , url = a.url + } + , Cmd.none + ) + + +isValid : Model -> Bool +isValid model = + model.url /= Nothing + + +get : Model -> Maybe Addon +get model = + let + a = + model.addon + in + if isValid model then + Just + { a + | url = model.url + } + + else + Nothing + + +type Msg + = SetUrl String + + +update : Flags -> Msg -> Model -> ( Model, Cmd Msg ) +update _ msg model = + case msg of + SetUrl url -> + ( { model | url = Util.Maybe.fromString url }, Cmd.none ) + + + +--- View + + +view : Texts -> Model -> Html Msg +view texts model = + div + [ class "flex flex-col" ] + [ div [ class "mb-4" ] + [ label + [ class S.inputLabel + ] + [ text texts.addonUrl + , B.inputRequired + ] + , input + [ type_ "text" + , placeholder texts.addonUrlPlaceholder + , class S.textInput + , classList [ ( "disabled", model.addon.id /= "" ) ] + , value (model.url |> Maybe.withDefault "") + , onInput SetUrl + , disabled (model.addon.id /= "") + ] + [] + , span [ class "text-sm opacity-75" ] + [ text texts.installInfoText + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/AddonArchiveManage.elm b/modules/webapp/src/main/elm/Comp/AddonArchiveManage.elm new file mode 100644 index 00000000..68f460f5 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonArchiveManage.elm @@ -0,0 +1,429 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonArchiveManage exposing (Model, Msg, addonInstallResult, init, loadAddons, update, view) + +import Api +import Api.Model.Addon exposing (Addon) +import Api.Model.AddonList exposing (AddonList) +import Api.Model.AddonRegister exposing (AddonRegister) +import Api.Model.BasicResult exposing (BasicResult) +import Comp.AddonArchiveForm +import Comp.AddonArchiveTable +import Comp.Basic as B +import Comp.ItemDetail.Model exposing (Msg(..)) +import Comp.MenuBar as MB +import Data.Flags exposing (Flags) +import Data.ServerEvent exposing (AddonInfo) +import Data.UiSettings exposing (UiSettings) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Http +import Markdown +import Messages.Comp.AddonArchiveManage exposing (Texts) +import Page exposing (Page(..)) +import Styles as S + + +type FormError + = FormErrorNone + | FormErrorHttp Http.Error + | FormErrorInvalid + | FormErrorSubmit String + + +type ViewMode + = Table + | Form + + +type DeleteConfirm + = DeleteConfirmOff + | DeleteConfirmOn + + +type alias Model = + { viewMode : ViewMode + , addons : List Addon + , formModel : Comp.AddonArchiveForm.Model + , loading : Bool + , formError : FormError + , deleteConfirm : DeleteConfirm + } + + +init : Flags -> ( Model, Cmd Msg ) +init flags = + let + ( fm, fc ) = + Comp.AddonArchiveForm.init + in + ( { viewMode = Table + , addons = [] + , formModel = fm + , loading = False + , formError = FormErrorNone + , deleteConfirm = DeleteConfirmOff + } + , Cmd.batch + [ Cmd.map FormMsg fc + , Api.addonsGetAll flags LoadAddonsResp + ] + ) + + +type Msg + = LoadAddons + | TableMsg Comp.AddonArchiveTable.Msg + | FormMsg Comp.AddonArchiveForm.Msg + | InitNewAddon + | SetViewMode ViewMode + | Submit + | RequestDelete + | CancelDelete + | DeleteAddonNow String + | LoadAddonsResp (Result Http.Error AddonList) + | AddAddonResp (Result Http.Error BasicResult) + | UpdateAddonResp (Result Http.Error BasicResult) + | DeleteAddonResp (Result Http.Error BasicResult) + | AddonInstallResp AddonInfo + + +loadAddons : Msg +loadAddons = + LoadAddons + + +addonInstallResult : AddonInfo -> Msg +addonInstallResult info = + AddonInstallResp info + + + +--- update + + +update : Flags -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg ) +update flags msg model = + case msg of + InitNewAddon -> + let + ( bm, bc ) = + Comp.AddonArchiveForm.init + + nm = + { model + | viewMode = Form + , formError = FormErrorNone + , formModel = bm + } + in + ( nm, Cmd.map FormMsg bc, Sub.none ) + + SetViewMode vm -> + ( { model | viewMode = vm, formError = FormErrorNone } + , if vm == Table then + Api.addonsGetAll flags LoadAddonsResp + + else + Cmd.none + , Sub.none + ) + + FormMsg lm -> + let + ( fm, fc ) = + Comp.AddonArchiveForm.update flags lm model.formModel + in + ( { model | formModel = fm, formError = FormErrorNone } + , Cmd.map FormMsg fc + , Sub.none + ) + + TableMsg lm -> + let + action = + Comp.AddonArchiveTable.update lm + in + case action of + Comp.AddonArchiveTable.Selected addon -> + let + ( bm, bc ) = + Comp.AddonArchiveForm.initWith addon + in + ( { model + | viewMode = Form + , formError = FormErrorNone + , formModel = bm + } + , Cmd.map FormMsg bc + , Sub.none + ) + + RequestDelete -> + ( { model | deleteConfirm = DeleteConfirmOn }, Cmd.none, Sub.none ) + + CancelDelete -> + ( { model | deleteConfirm = DeleteConfirmOff }, Cmd.none, Sub.none ) + + DeleteAddonNow id -> + ( { model | deleteConfirm = DeleteConfirmOff, loading = True } + , Api.addonsDelete flags id DeleteAddonResp + , Sub.none + ) + + LoadAddons -> + ( { model | loading = True } + , Api.addonsGetAll flags LoadAddonsResp + , Sub.none + ) + + LoadAddonsResp (Ok list) -> + ( { model | loading = False, addons = list.items, formError = FormErrorNone } + , Cmd.none + , Sub.none + ) + + LoadAddonsResp (Err err) -> + ( { model | loading = False, formError = FormErrorHttp err }, Cmd.none, Sub.none ) + + AddonInstallResp info -> + if info.success then + ( { model | loading = False, viewMode = Table }, Api.addonsGetAll flags LoadAddonsResp, Sub.none ) + + else + ( { model | loading = False, formError = FormErrorSubmit info.message }, Cmd.none, Sub.none ) + + Submit -> + case Comp.AddonArchiveForm.get model.formModel of + Just data -> + if data.id /= "" then + ( { model | loading = True } + , Api.addonsUpdate flags data.id UpdateAddonResp + , Sub.none + ) + + else + ( { model | loading = True } + , Api.addonsInstall + flags + (AddonRegister <| Maybe.withDefault "" data.url) + AddAddonResp + , Sub.none + ) + + Nothing -> + ( { model | formError = FormErrorInvalid }, Cmd.none, Sub.none ) + + AddAddonResp (Ok res) -> + if res.success then + ( model, Cmd.none, Sub.none ) + + else + ( { model | loading = False, formError = FormErrorSubmit res.message }, Cmd.none, Sub.none ) + + AddAddonResp (Err err) -> + ( { model | loading = False, formError = FormErrorHttp err }, Cmd.none, Sub.none ) + + UpdateAddonResp (Ok res) -> + if res.success then + ( model, Cmd.none, Sub.none ) + + else + ( { model | loading = False, formError = FormErrorSubmit res.message }, Cmd.none, Sub.none ) + + UpdateAddonResp (Err err) -> + ( { model | loading = False, formError = FormErrorHttp err }, Cmd.none, Sub.none ) + + DeleteAddonResp (Ok res) -> + if res.success then + update flags (SetViewMode Table) { model | loading = False } + + else + ( { model | formError = FormErrorSubmit res.message, loading = False }, Cmd.none, Sub.none ) + + DeleteAddonResp (Err err) -> + ( { model | formError = FormErrorHttp err, loading = False }, Cmd.none, Sub.none ) + + + +--- view + + +view : Texts -> UiSettings -> Flags -> Model -> Html Msg +view texts settings flags model = + if model.viewMode == Table then + viewTable texts model + + else + viewForm texts settings flags model + + +viewTable : Texts -> Model -> Html Msg +viewTable texts model = + div [ class "flex flex-col" ] + [ MB.view + { start = + [] + , end = + [ MB.PrimaryButton + { tagger = InitNewAddon + , title = texts.createNewAddonArchive + , icon = Just "fa fa-plus" + , label = texts.newAddonArchive + } + ] + , rootClasses = "mb-4" + , sticky = True + } + , div + [ class "flex flex-col" + ] + [ Html.map TableMsg + (Comp.AddonArchiveTable.view texts.addonArchiveTable model.addons) + ] + , B.loadingDimmer + { label = "" + , active = model.loading + } + ] + + +viewForm : Texts -> UiSettings -> Flags -> Model -> Html Msg +viewForm texts _ _ model = + let + newAddon = + model.formModel.addon.id == "" + + isValid = + Comp.AddonArchiveForm.get model.formModel /= Nothing + in + div [ class "relative" ] + [ Html.form [] + [ if newAddon then + h1 [ class S.header2 ] + [ text texts.createNewAddonArchive + ] + + else + h1 [ class S.header2 ] + [ text (Comp.AddonArchiveForm.get model.formModel |> Maybe.map .name |> Maybe.withDefault "Update") + ] + , MB.view + { start = + [ MB.SecondaryButton + { tagger = SetViewMode Table + , title = texts.basics.backToList + , icon = Just "fa fa-arrow-left" + , label = texts.basics.back + } + ] + , end = + if not newAddon then + [ MB.DeleteButton + { tagger = RequestDelete + , title = texts.deleteThisAddonArchive + , icon = Just "fa fa-trash" + , label = texts.basics.delete + } + ] + + else + [] + , rootClasses = "mb-4" + , sticky = True + } + , div + [ classList + [ ( "hidden", model.formError == FormErrorNone ) + ] + , class "my-2" + , class S.errorMessage + ] + [ case model.formError of + FormErrorNone -> + text "" + + FormErrorHttp err -> + text (texts.httpError err) + + FormErrorInvalid -> + text texts.correctFormErrors + + FormErrorSubmit m -> + text m + ] + , div [] + [ Html.map FormMsg (Comp.AddonArchiveForm.view texts.addonArchiveForm model.formModel) + ] + , MB.view + { start = + [ MB.PrimaryButton + { tagger = Submit + , title = texts.installNow + , icon = + if newAddon then + Just "fa fa-save" + + else + Just "fa fa-arrows-rotate" + , label = + if newAddon then + texts.installNow + + else + texts.updateNow + } + ] + , end = [] + , rootClasses = "mb-4" + , sticky = False + } + , div + [ class "mb-4" + , classList [ ( "hidden", newAddon ) ] + ] + [ label [ class S.inputLabel ] [ text texts.description ] + , case model.formModel.addon.description of + Just desc -> + Markdown.toHtml [ class "markdown-preview" ] desc + + Nothing -> + div [ class "italic" ] [ text "-" ] + ] + , B.loadingDimmer + { active = model.loading + , label = texts.basics.loading + } + , B.contentDimmer + (model.deleteConfirm == DeleteConfirmOn) + (div [ class "flex flex-col" ] + [ div [ class "text-lg" ] + [ i [ class "fa fa-info-circle mr-2" ] [] + , text texts.reallyDeleteAddonArchive + ] + , div [ class "mt-4 flex flex-row items-center" ] + [ B.deleteButton + { label = texts.basics.yes + , icon = "fa fa-check" + , disabled = False + , handler = onClick (DeleteAddonNow model.formModel.addon.id) + , attrs = [ href "#" ] + } + , B.secondaryButton + { label = texts.basics.no + , icon = "fa fa-times" + , disabled = False + , handler = onClick CancelDelete + , attrs = [ href "#", class "ml-2" ] + } + ] + ] + ) + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/AddonArchiveTable.elm b/modules/webapp/src/main/elm/Comp/AddonArchiveTable.elm new file mode 100644 index 00000000..4b58cb81 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonArchiveTable.elm @@ -0,0 +1,72 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonArchiveTable exposing (..) + +import Api.Model.Addon exposing (Addon) +import Comp.Basic as B +import Html exposing (Html, div, table, tbody, td, text, th, thead, tr) +import Html.Attributes exposing (class) +import Messages.Comp.AddonArchiveTable exposing (Texts) +import Styles as S + + +type Msg + = SelectAddon Addon + + +type TableAction + = Selected Addon + + + +--- Update + + +update : Msg -> TableAction +update msg = + case msg of + SelectAddon addon -> + Selected addon + + + +--- View + + +view : Texts -> List Addon -> Html Msg +view texts addons = + table [ class S.tableMain ] + [ thead [] + [ tr [] + [ th [ class "" ] [] + , th [ class "text-left" ] + [ text texts.basics.name + ] + , th [ class "text-left" ] + [ text texts.version + ] + ] + ] + , tbody [] + (List.map (renderAddonLine texts) addons) + ] + + +renderAddonLine : Texts -> Addon -> Html Msg +renderAddonLine texts addon = + tr + [ class S.tableRow + ] + [ B.editLinkTableCell texts.basics.edit (SelectAddon addon) + , td [ class "text-left py-4 md:py-2" ] + [ text addon.name + ] + , td [ class "text-left" ] + [ text addon.version + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/AddonRunConfigForm.elm b/modules/webapp/src/main/elm/Comp/AddonRunConfigForm.elm new file mode 100644 index 00000000..0ad60d8f --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonRunConfigForm.elm @@ -0,0 +1,709 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonRunConfigForm exposing (Model, Msg, get, init, initWith, update, view) + +import Api +import Api.Model.Addon exposing (Addon) +import Api.Model.AddonList exposing (AddonList) +import Api.Model.AddonRef exposing (AddonRef) +import Api.Model.AddonRunConfig exposing (AddonRunConfig) +import Api.Model.User exposing (User) +import Api.Model.UserList exposing (UserList) +import Comp.Basic as B +import Comp.CalEventInput +import Comp.Dropdown +import Comp.MenuBar as MB +import Data.AddonTrigger exposing (AddonTrigger) +import Data.CalEvent exposing (CalEvent) +import Data.DropdownStyle as DS +import Data.Flags exposing (Flags) +import Data.TimeZone exposing (TimeZone) +import Data.UiSettings exposing (UiSettings) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onInput) +import Http +import Markdown +import Messages.Comp.AddonRunConfigForm exposing (Texts) +import Process +import Styles as S +import Task +import Util.List +import Util.String + + +type alias Model = + { runConfig : AddonRunConfig + , name : String + , enabled : Bool + , userDropdown : Comp.Dropdown.Model User + , userId : Maybe String + , userList : List User + , scheduleModel : Maybe Comp.CalEventInput.Model + , schedule : Maybe CalEvent + , triggerDropdown : Comp.Dropdown.Model AddonTrigger + , addons : List AddonRef + , selectedAddon : Maybe AddonConfigModel + , existingAddonDropdown : Comp.Dropdown.Model Addon + , existingAddons : List Addon + , configApplied : Bool + } + + +type alias AddonConfigModel = + { ref : AddonRef + , position : Int + , args : String + , readMore : Bool + } + + +getRef : AddonConfigModel -> AddonRef +getRef cfg = + let + a = + cfg.ref + in + { a | args = cfg.args } + + +emptyModel : Model +emptyModel = + { runConfig = Api.Model.AddonRunConfig.empty + , name = "" + , enabled = True + , userDropdown = Comp.Dropdown.makeSingle + , userId = Nothing + , userList = [] + , scheduleModel = Nothing + , schedule = Nothing + , triggerDropdown = + Comp.Dropdown.makeMultipleList + { options = Data.AddonTrigger.all, selected = [] } + , addons = [] + , selectedAddon = Nothing + , existingAddonDropdown = Comp.Dropdown.makeSingle + , existingAddons = [] + , configApplied = False + } + + +init : Flags -> ( Model, Cmd Msg ) +init flags = + ( emptyModel + , Cmd.batch + [ Api.getUsers flags UserListResp + , Api.addonsGetAll flags AddonListResp + ] + ) + + +initWith : Flags -> AddonRunConfig -> ( Model, Cmd Msg ) +initWith flags a = + let + ce = + Maybe.andThen Data.CalEvent.fromEvent a.schedule + + ceInit = + Maybe.map (Comp.CalEventInput.init flags) ce + + triggerModel = + Comp.Dropdown.makeMultipleList + { options = Data.AddonTrigger.all + , selected = Data.AddonTrigger.fromList a.trigger + } + in + ( { emptyModel + | runConfig = a + , name = a.name + , enabled = a.enabled + , scheduleModel = Maybe.map Tuple.first ceInit + , schedule = ce + , triggerDropdown = triggerModel + , userId = a.userId + , addons = a.addons + } + , Cmd.batch + [ Api.getUsers flags UserListResp + , Api.addonsGetAll flags AddonListResp + , Maybe.map Tuple.second ceInit + |> Maybe.map (Cmd.map ScheduleMsg) + |> Maybe.withDefault Cmd.none + ] + ) + + +isValid : Model -> Bool +isValid model = + model.name + /= "" + && (Comp.Dropdown.getSelected model.triggerDropdown + |> List.isEmpty + |> not + ) + && (List.isEmpty model.addons + |> not + ) + + +get : Model -> Maybe AddonRunConfig +get model = + let + a = + model.runConfig + in + if isValid model then + Just + { a + | name = model.name + , enabled = model.enabled + , schedule = Maybe.map Data.CalEvent.makeEvent model.schedule + , trigger = + Comp.Dropdown.getSelected model.triggerDropdown + |> List.map Data.AddonTrigger.asString + , userId = model.userId + , addons = model.addons + } + + else + Nothing + + +type Msg + = SetName String + | UserListResp (Result Http.Error UserList) + | AddonListResp (Result Http.Error AddonList) + | ScheduleMsg Comp.CalEventInput.Msg + | UserDropdownMsg (Comp.Dropdown.Msg User) + | TriggerDropdownMsg (Comp.Dropdown.Msg AddonTrigger) + | AddonDropdownMsg (Comp.Dropdown.Msg Addon) + | Configure Int AddonRef + | Up Int + | Down Int + | Remove Int + | ToggleEnabled + | ConfigSetArgs String + | ConfigApply + | ConfigCancel + | AddSelectedAddon + | ConfigToggleReadMore + | ConfigArgsUpdated Bool + + + +--- Update + + +update : Flags -> TimeZone -> Msg -> Model -> ( Model, Cmd Msg ) +update flags tz msg model = + case msg of + UserListResp (Ok list) -> + let + um = + Comp.Dropdown.makeSingleList + { options = list.items + , selected = Nothing + } + in + ( { model | userDropdown = um, userList = list.items }, Cmd.none ) + + UserListResp (Err err) -> + ( model, Cmd.none ) + + AddonListResp (Ok list) -> + let + am = + Comp.Dropdown.makeSingleList + { options = list.items + , selected = Nothing + } + in + ( { model | existingAddonDropdown = am, existingAddons = list.items }, Cmd.none ) + + AddonListResp (Err err) -> + ( model, Cmd.none ) + + UserDropdownMsg lm -> + let + ( um, cmd ) = + Comp.Dropdown.update lm model.userDropdown + + sel = + Comp.Dropdown.getSelected um |> List.head + in + ( { model | userDropdown = um, userId = Maybe.map .id sel }, Cmd.map UserDropdownMsg cmd ) + + TriggerDropdownMsg lm -> + let + ( tm, tc ) = + Comp.Dropdown.update lm model.triggerDropdown + + ( nm, nc ) = + initScheduleIfNeeded flags { model | triggerDropdown = tm } tz + in + ( nm, Cmd.batch [ Cmd.map TriggerDropdownMsg tc, nc ] ) + + ScheduleMsg lm -> + case model.scheduleModel of + Just m -> + let + ( cm, cc, ce ) = + Comp.CalEventInput.update flags tz model.schedule lm m + in + ( { model | scheduleModel = Just cm, schedule = ce }, Cmd.map ScheduleMsg cc ) + + Nothing -> + ( model, Cmd.none ) + + ToggleEnabled -> + ( { model | enabled = not model.enabled }, Cmd.none ) + + AddonDropdownMsg lm -> + let + ( am, ac ) = + Comp.Dropdown.update lm model.existingAddonDropdown + in + ( { model | existingAddonDropdown = am }, Cmd.map AddonDropdownMsg ac ) + + Configure index ref -> + let + cfg = + { ref = ref + , position = index + 1 + , args = ref.args + , readMore = False + } + in + ( { model | selectedAddon = Just cfg }, Cmd.none ) + + ConfigCancel -> + ( { model | selectedAddon = Nothing }, Cmd.none ) + + ConfigToggleReadMore -> + case model.selectedAddon of + Just cfg -> + ( { model | selectedAddon = Just { cfg | readMore = not cfg.readMore } }, Cmd.none ) + + Nothing -> + ( model, Cmd.none ) + + ConfigArgsUpdated flag -> + ( { model | configApplied = flag }, Cmd.none ) + + ConfigSetArgs str -> + case model.selectedAddon of + Just cfg -> + ( { model | selectedAddon = Just { cfg | args = str } } + , Cmd.none + ) + + Nothing -> + ( model, Cmd.none ) + + ConfigApply -> + case model.selectedAddon of + Just cfg -> + let + na = + getRef cfg + + addons = + Util.List.replaceByIndex (cfg.position - 1) na model.addons + in + ( { model | addons = addons, configApplied = True } + , Process.sleep 1200 |> Task.perform (\_ -> ConfigArgsUpdated False) + ) + + Nothing -> + ( model, Cmd.none ) + + AddSelectedAddon -> + let + sel = + Comp.Dropdown.getSelected model.existingAddonDropdown |> List.head + + ( dm, _ ) = + Comp.Dropdown.update (Comp.Dropdown.SetSelection []) model.existingAddonDropdown + + addon = + Maybe.map + (\a -> + { addonId = a.id + , name = a.name + , version = a.version + , description = a.description + , args = "" + } + ) + sel + + newAddons = + Maybe.map (\e -> e :: model.addons) addon + |> Maybe.withDefault model.addons + in + ( { model | addons = newAddons, existingAddonDropdown = dm, selectedAddon = Nothing }, Cmd.none ) + + Up curIndex -> + let + newAddons = + Util.List.changePosition curIndex (curIndex - 1) model.addons + in + ( { model | addons = newAddons, selectedAddon = Nothing }, Cmd.none ) + + Down curIndex -> + let + newAddons = + Util.List.changePosition (curIndex + 1) curIndex model.addons + in + ( { model | addons = newAddons, selectedAddon = Nothing }, Cmd.none ) + + SetName str -> + ( { model | name = str }, Cmd.none ) + + Remove index -> + ( { model | addons = Util.List.removeByIndex index model.addons, selectedAddon = Nothing }, Cmd.none ) + + +initScheduleIfNeeded : Flags -> Model -> TimeZone -> ( Model, Cmd Msg ) +initScheduleIfNeeded flags model tz = + let + hasTrigger = + Comp.Dropdown.getSelected model.triggerDropdown + |> List.any ((==) Data.AddonTrigger.Scheduled) + + noModel = + model.scheduleModel == Nothing + + hasModel = + not noModel + + ce = + Data.CalEvent.everyMonthTz tz + + ( cm, cc ) = + Comp.CalEventInput.init flags ce + in + if hasTrigger && noModel then + ( { model | scheduleModel = Just cm, schedule = Just ce }, Cmd.map ScheduleMsg cc ) + + else if not hasTrigger && hasModel then + ( { model | scheduleModel = Nothing, schedule = Nothing }, Cmd.none ) + + else + ( model, Cmd.none ) + + + +--- View + + +view : Texts -> UiSettings -> Model -> Html Msg +view texts settings model = + let + userDs = + { makeOption = \user -> { text = user.login, additional = "" } + , placeholder = texts.basics.selectPlaceholder + , labelColor = \_ -> \_ -> "" + , style = DS.mainStyle + } + + triggerDs = + { makeOption = \trigger -> { text = Data.AddonTrigger.asString trigger, additional = "" } + , placeholder = texts.basics.selectPlaceholder + , labelColor = \_ -> \_ -> "" + , style = DS.mainStyle + } + in + div + [ class "flex flex-col" ] + [ div [ class "mb-4" ] + [ div [ class "mb-4" ] + [ label + [ class S.inputLabel + ] + [ text texts.basics.name + , B.inputRequired + ] + , input + [ type_ "text" + , placeholder texts.chooseName + , value model.name + , onInput SetName + , class S.textInput + , classList [ ( S.inputErrorBorder, model.name == "" ) ] + ] + [] + ] + , div [ class "mb-4" ] + [ MB.viewItem <| + MB.Checkbox + { tagger = \_ -> ToggleEnabled + , label = texts.enableDisable + , value = model.enabled + , id = "addon-run-config-enabled" + } + ] + , div [ class "mb-4" ] + [ label + [ class S.inputLabel + ] + [ text texts.impersonateUser + ] + , Html.map UserDropdownMsg + (Comp.Dropdown.view2 userDs settings model.userDropdown) + ] + , div [ class "mb-4" ] + [ label + [ class S.inputLabel + ] + [ text texts.triggerRun + , B.inputRequired + ] + , Html.map TriggerDropdownMsg + (Comp.Dropdown.view2 triggerDs settings model.triggerDropdown) + ] + , case model.scheduleModel of + Nothing -> + span [ class "hidden" ] [] + + Just m -> + div [ class "mb-4" ] + [ label + [ class S.inputLabel ] + [ text texts.schedule + ] + , Html.map ScheduleMsg (Comp.CalEventInput.view2 texts.calEventInput "" model.schedule m) + ] + ] + , div [ class "mb-4" ] + [ h2 [ class S.header2 ] + [ text texts.addons ] + , addonRef texts model + , div [ class "mb-4" ] + [ label [ class S.inputLabel ] + [ text texts.includedAddons + , B.inputRequired + ] + , newAddon texts settings model + , div [ class "mb-4" ] + [ div [ class "flex flex-col mb-4" ] + (List.indexedMap (addonLine texts model) model.addons) + ] + ] + ] + ] + + +newAddon : Texts -> UiSettings -> Model -> Html Msg +newAddon texts uiSettings model = + let + addonDs = + { makeOption = \addon -> { text = addon.name ++ " / " ++ addon.version, additional = "" } + , placeholder = texts.basics.selectPlaceholder + , labelColor = \_ -> \_ -> "" + , style = DS.mainStyle + } + in + div [ class "mb-4" ] + [ div [ class "flex flex-row" ] + [ div [ class "flex-grow mr-2" ] + [ Html.map AddonDropdownMsg + (Comp.Dropdown.view2 addonDs uiSettings model.existingAddonDropdown) + ] + , B.primaryBasicButton + { label = texts.add + , icon = "fa fa-plus" + , disabled = List.isEmpty (Comp.Dropdown.getSelected model.existingAddonDropdown) + , handler = onClick AddSelectedAddon + , attrs = [ href "#" ] + } + ] + ] + + +addonRef : Texts -> Model -> Html Msg +addonRef texts model = + let + maybeRef = + Maybe.map .ref model.selectedAddon + + refInfo = + case model.selectedAddon of + Nothing -> + div [ class "mb-4" ] + [ text "[ -- ]" + ] + + Just cfg -> + let + ( descr, requireFolding ) = + case cfg.ref.description of + Just d -> + let + part = + Util.String.firstSentenceOrMax 120 d + + text = + if cfg.readMore then + d + + else + Maybe.withDefault d part + in + ( Markdown.toHtml [ class "markdown-preview" ] text, part /= Nothing ) + + Nothing -> + ( span [ class "italic" ] [ text "No description." ], False ) + in + div [ class "flex flex-col mb-4" ] + [ div [ class "mt-2" ] + [ label [ class " font-semibold py-0.5 " ] + [ text cfg.ref.name + , text " " + , text cfg.ref.version + , text " (pos. " + , text <| String.fromInt cfg.position + , text ")" + , span + [ classList [ ( "hidden", not requireFolding ) ] + , class "ml-2" + ] + [ a + [ class "px-4" + , class S.link + , href "#" + , onClick ConfigToggleReadMore + ] + [ if cfg.readMore then + text texts.readLess + + else + text texts.readMore + ] + ] + ] + , div [ class "px-3 py-1 border-l dark:border-slate-600" ] + [ descr + ] + ] + ] + in + div + [ class "flex flex-col mb-3" + , classList [ ( "disabled", maybeRef == Nothing ) ] + ] + [ refInfo + , div [ class "mb-2" ] + [ label [ class S.inputLabel ] [ text texts.arguments ] + , textarea + [ Maybe.map .args model.selectedAddon |> Maybe.withDefault "" |> value + , class S.textAreaInput + , class "font-mono" + , rows 8 + , onInput ConfigSetArgs + ] + [] + ] + , MB.view + { start = + [ MB.PrimaryButton + { tagger = ConfigApply + , title = "" + , icon = Just "fa fa-save" + , label = texts.update + } + , MB.SecondaryButton + { tagger = ConfigCancel + , title = texts.basics.cancel + , icon = Just "fa fa-times" + , label = texts.basics.cancel + } + , MB.CustomElement <| + div + [ classList [ ( "hidden", not model.configApplied ) ] + , class S.successText + , class "inline-block min-w-fit font-semibold text-normal min-w-fit" + ] + [ text texts.argumentsUpdated + , i [ class "fa fa-thumbs-up ml-2" ] [] + ] + ] + , end = [] + , rootClasses = "mb-4 text-sm" + , sticky = False + } + ] + + +addonLine : Texts -> Model -> Int -> AddonRef -> Html Msg +addonLine texts model index ref = + let + isSelected = + case model.selectedAddon of + Just cfg -> + cfg.position - 1 == index + + Nothing -> + False + in + div + [ class "flex flex-row items-center px-4 py-4 rounded shadow dark:border dark:border-slate-600 mb-2" + , classList [ ( "ring-2", isSelected ) ] + ] + [ div [ class "px-2 hidden sm:block" ] + [ span [ class "label rounded-full opacity-75" ] + [ text <| String.fromInt (index + 1) + ] + ] + , div [ class "px-4 font-semibold" ] + [ text ref.name + , text " v" + , text ref.version + ] + , div [ class "flex-grow" ] + [] + , div [ class "px-2" ] + [ MB.view + { start = [] + , end = + [ MB.PrimaryButton + { tagger = Configure index ref + , title = texts.configureTitle + , icon = Just "fa fa-cog" + , label = texts.configureLabel + } + , MB.CustomElement <| + B.secondaryButton + { handler = onClick (Up index) + , attrs = [ title "Move up", href "#" ] + , icon = "fa fa-arrow-up" + , label = "" + , disabled = index == 0 + } + , MB.CustomElement <| + B.secondaryButton + { handler = onClick (Down index) + , attrs = [ title "Move down", href "#" ] + , icon = "fa fa-arrow-down" + , label = "" + , disabled = index + 1 == List.length model.addons + } + , MB.CustomElement <| + B.deleteButton + { label = "" + , icon = "fa fa-trash" + , disabled = False + , handler = onClick (Remove index) + , attrs = [ href "#" ] + } + ] + , rootClasses = "text-sm" + , sticky = False + } + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/AddonRunConfigManage.elm b/modules/webapp/src/main/elm/Comp/AddonRunConfigManage.elm new file mode 100644 index 00000000..641b821d --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonRunConfigManage.elm @@ -0,0 +1,364 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonRunConfigManage exposing (Model, Msg, init, loadConfigs, update, view) + +import Api +import Api.Model.AddonRunConfig exposing (AddonRunConfig) +import Api.Model.AddonRunConfigList exposing (AddonRunConfigList) +import Api.Model.BasicResult exposing (BasicResult) +import Comp.AddonRunConfigForm +import Comp.AddonRunConfigTable +import Comp.Basic as B +import Comp.ItemDetail.Model exposing (Msg(..)) +import Comp.MenuBar as MB +import Data.Flags exposing (Flags) +import Data.TimeZone exposing (TimeZone) +import Data.UiSettings exposing (UiSettings) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Http +import Messages.Comp.AddonRunConfigManage exposing (Texts) +import Page exposing (Page(..)) +import Styles as S + + +type FormError + = FormErrorNone + | FormErrorHttp Http.Error + | FormErrorInvalid + | FormErrorSubmit String + + +type ViewMode + = Table + | Form + + +type DeleteConfirm + = DeleteConfirmOff + | DeleteConfirmOn + + +type alias Model = + { viewMode : ViewMode + , runConfigs : List AddonRunConfig + , formModel : Comp.AddonRunConfigForm.Model + , loading : Bool + , formError : FormError + , deleteConfirm : DeleteConfirm + } + + +init : Flags -> ( Model, Cmd Msg ) +init flags = + let + ( fm, fc ) = + Comp.AddonRunConfigForm.init flags + in + ( { viewMode = Table + , runConfigs = [] + , formModel = fm + , loading = False + , formError = FormErrorNone + , deleteConfirm = DeleteConfirmOff + } + , Cmd.batch + [ Cmd.map FormMsg fc + , Api.addonRunConfigGet flags LoadConfigsResp + ] + ) + + +type Msg + = LoadRunConfigs + | TableMsg Comp.AddonRunConfigTable.Msg + | FormMsg Comp.AddonRunConfigForm.Msg + | InitNewConfig + | SetViewMode ViewMode + | Submit + | RequestDelete + | CancelDelete + | DeleteConfigNow String + | LoadConfigsResp (Result Http.Error AddonRunConfigList) + | AddConfigResp (Result Http.Error BasicResult) + | DeleteConfigResp (Result Http.Error BasicResult) + + +loadConfigs : Msg +loadConfigs = + LoadRunConfigs + + + +--- update + + +update : Flags -> TimeZone -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg ) +update flags tz msg model = + case msg of + InitNewConfig -> + let + ( bm, bc ) = + Comp.AddonRunConfigForm.init flags + + nm = + { model + | viewMode = Form + , formError = FormErrorNone + , formModel = bm + } + in + ( nm, Cmd.map FormMsg bc, Sub.none ) + + SetViewMode vm -> + ( { model | viewMode = vm, formError = FormErrorNone } + , if vm == Table then + Api.addonRunConfigGet flags LoadConfigsResp + + else + Cmd.none + , Sub.none + ) + + FormMsg lm -> + let + ( fm, fc ) = + Comp.AddonRunConfigForm.update flags tz lm model.formModel + in + ( { model | formModel = fm, formError = FormErrorNone } + , Cmd.map FormMsg fc + , Sub.none + ) + + TableMsg lm -> + let + action = + Comp.AddonRunConfigTable.update lm + in + case action of + Comp.AddonRunConfigTable.Selected addon -> + let + ( bm, bc ) = + Comp.AddonRunConfigForm.initWith flags addon + in + ( { model + | viewMode = Form + , formError = FormErrorNone + , formModel = bm + } + , Cmd.map FormMsg bc + , Sub.none + ) + + RequestDelete -> + ( { model | deleteConfirm = DeleteConfirmOn }, Cmd.none, Sub.none ) + + CancelDelete -> + ( { model | deleteConfirm = DeleteConfirmOff }, Cmd.none, Sub.none ) + + DeleteConfigNow id -> + ( { model | deleteConfirm = DeleteConfirmOff, loading = True } + , Api.addonRunConfigDelete flags id DeleteConfigResp + , Sub.none + ) + + LoadRunConfigs -> + ( { model | loading = True } + , Api.addonRunConfigGet flags LoadConfigsResp + , Sub.none + ) + + LoadConfigsResp (Ok list) -> + ( { model | loading = False, runConfigs = list.items, formError = FormErrorNone } + , Cmd.none + , Sub.none + ) + + LoadConfigsResp (Err err) -> + ( { model | loading = False, formError = FormErrorHttp err }, Cmd.none, Sub.none ) + + Submit -> + case Comp.AddonRunConfigForm.get model.formModel of + Just data -> + ( { model | loading = True }, Api.addonRunConfigSet flags data AddConfigResp, Sub.none ) + + Nothing -> + ( { model | formError = FormErrorInvalid }, Cmd.none, Sub.none ) + + AddConfigResp (Ok res) -> + if res.success then + ( { model | loading = False }, Cmd.none, Sub.none ) + + else + ( { model | loading = False, formError = FormErrorSubmit res.message }, Cmd.none, Sub.none ) + + AddConfigResp (Err err) -> + ( { model | loading = False, formError = FormErrorHttp err }, Cmd.none, Sub.none ) + + DeleteConfigResp (Ok res) -> + if res.success then + update flags tz (SetViewMode Table) { model | loading = False } + + else + ( { model | formError = FormErrorSubmit res.message, loading = False }, Cmd.none, Sub.none ) + + DeleteConfigResp (Err err) -> + ( { model | formError = FormErrorHttp err, loading = False }, Cmd.none, Sub.none ) + + + +--- view + + +view : Texts -> UiSettings -> Flags -> Model -> Html Msg +view texts settings flags model = + if model.viewMode == Table then + viewTable texts model + + else + viewForm texts settings flags model + + +viewTable : Texts -> Model -> Html Msg +viewTable texts model = + div [ class "flex flex-col" ] + [ MB.view + { start = + [] + , end = + [ MB.PrimaryButton + { tagger = InitNewConfig + , title = texts.createNewAddonRunConfig + , icon = Just "fa fa-plus" + , label = texts.newAddonRunConfig + } + ] + , rootClasses = "mb-4" + , sticky = True + } + , div + [ class "flex flex-col" + ] + [ Html.map TableMsg + (Comp.AddonRunConfigTable.view texts.addonArchiveTable model.runConfigs) + ] + , B.loadingDimmer + { label = "" + , active = model.loading + } + ] + + +viewForm : Texts -> UiSettings -> Flags -> Model -> Html Msg +viewForm texts uiSettings _ model = + let + newConfig = + model.formModel.runConfig.id == "" + + isValid = + Comp.AddonRunConfigForm.get model.formModel /= Nothing + in + div [] + [ Html.form [] + [ if newConfig then + h1 [ class S.header2 ] + [ text texts.createNewAddonRunConfig + ] + + else + h1 [ class S.header2 ] + [ text (Comp.AddonRunConfigForm.get model.formModel |> Maybe.map .name |> Maybe.withDefault "Update") + ] + , MB.view + { start = + [ MB.CustomElement <| + B.primaryButton + { handler = onClick Submit + , title = texts.basics.submitThisForm + , icon = "fa fa-save" + , label = texts.basics.submit + , disabled = not isValid + , attrs = [ href "#" ] + } + , MB.SecondaryButton + { tagger = SetViewMode Table + , title = texts.basics.backToList + , icon = Just "fa fa-arrow-left" + , label = texts.basics.back + } + ] + , end = + if not newConfig then + [ MB.DeleteButton + { tagger = RequestDelete + , title = texts.deleteThisAddonRunConfig + , icon = Just "fa fa-trash" + , label = texts.basics.delete + } + ] + + else + [] + , rootClasses = "mb-4" + , sticky = True + } + , div + [ classList + [ ( "hidden", model.formError == FormErrorNone ) + ] + , class "my-2" + , class S.errorMessage + ] + [ case model.formError of + FormErrorNone -> + text "" + + FormErrorHttp err -> + text (texts.httpError err) + + FormErrorInvalid -> + text texts.correctFormErrors + + FormErrorSubmit m -> + text m + ] + , div [] + [ Html.map FormMsg (Comp.AddonRunConfigForm.view texts.addonArchiveForm uiSettings model.formModel) + ] + , B.loadingDimmer + { active = model.loading + , label = texts.basics.loading + } + , B.contentDimmer + (model.deleteConfirm == DeleteConfirmOn) + (div [ class "flex flex-col" ] + [ div [ class "text-lg" ] + [ i [ class "fa fa-info-circle mr-2" ] [] + , text texts.reallyDeleteAddonRunConfig + ] + , div [ class "mt-4 flex flex-row items-center" ] + [ B.deleteButton + { label = texts.basics.yes + , icon = "fa fa-check" + , disabled = False + , handler = onClick (DeleteConfigNow model.formModel.runConfig.id) + , attrs = [ href "#" ] + } + , B.secondaryButton + { label = texts.basics.no + , icon = "fa fa-times" + , disabled = False + , handler = onClick CancelDelete + , attrs = [ href "#", class "ml-2" ] + } + ] + ] + ) + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/AddonRunConfigTable.elm b/modules/webapp/src/main/elm/Comp/AddonRunConfigTable.elm new file mode 100644 index 00000000..02febe38 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddonRunConfigTable.elm @@ -0,0 +1,79 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.AddonRunConfigTable exposing (..) + +import Api.Model.AddonRunConfig exposing (AddonRunConfig) +import Comp.Basic as B +import Html exposing (Html, div, table, tbody, td, text, th, thead, tr) +import Html.Attributes exposing (class) +import Messages.Comp.AddonRunConfigTable exposing (Texts) +import Styles as S +import Util.Html + + +type Msg + = SelectRunConfig AddonRunConfig + + +type TableAction + = Selected AddonRunConfig + + + +--- Update + + +update : Msg -> TableAction +update msg = + case msg of + SelectRunConfig cfg -> + Selected cfg + + + +--- View + + +view : Texts -> List AddonRunConfig -> Html Msg +view texts addons = + table [ class S.tableMain ] + [ thead [] + [ tr [] + [ th [ class "" ] [] + , th [ class "text-left" ] + [ text texts.basics.name + ] + , th [ class "px-2 text-center" ] [ text texts.enabled ] + , th [ class "px-2 text-left" ] [ text texts.trigger ] + , th [ class "px-2 text-center" ] [ text "# Addons" ] + ] + ] + , tbody [] + (List.map (renderRunConfigLine texts) addons) + ] + + +renderRunConfigLine : Texts -> AddonRunConfig -> Html Msg +renderRunConfigLine texts cfg = + tr + [ class S.tableRow + ] + [ B.editLinkTableCell texts.basics.edit (SelectRunConfig cfg) + , td [ class "text-left py-4 md:py-2" ] + [ text cfg.name + ] + , td [ class "w-px whitespace-nowrap px-2 text-center" ] + [ Util.Html.checkbox2 cfg.enabled + ] + , td [ class "px-2 text-left" ] + [ text (String.join ", " cfg.trigger) + ] + , td [ class "px-2 text-center" ] + [ text (String.fromInt <| List.length cfg.addons) + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm index 726166fb..e01db9d7 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/Model.elm @@ -28,6 +28,8 @@ module Comp.ItemDetail.Model exposing , resultModelCmdSub ) +import Api.Model.AddonRunConfig exposing (AddonRunConfig) +import Api.Model.AddonRunConfigList exposing (AddonRunConfigList) import Api.Model.BasicResult exposing (BasicResult) import Api.Model.CustomField exposing (CustomField) import Api.Model.EquipmentList exposing (EquipmentList) @@ -72,6 +74,7 @@ import Set exposing (Set) type alias Model = { item : ItemDetail + , runConfigs : List AddonRunConfig , visibleAttach : Int , attachMenuOpen : Bool , menuOpen : Bool @@ -123,6 +126,9 @@ type alias Model = , viewMode : ViewMode , showQrModel : ShowQrModel , itemLinkModel : Comp.ItemLinkForm.Model + , showRunAddon : Bool + , addonRunConfigDropdown : Comp.Dropdown.Model AddonRunConfig + , addonRunSubmitted : Bool } @@ -204,6 +210,7 @@ isEditNotes field = emptyModel : Model emptyModel = { item = Api.Model.ItemDetail.empty + , runConfigs = [] , visibleAttach = 0 , attachMenuOpen = False , menuOpen = False @@ -259,6 +266,9 @@ emptyModel = , viewMode = SimpleView , showQrModel = initShowQrModel , itemLinkModel = Comp.ItemLinkForm.emptyModel + , showRunAddon = False + , addonRunConfigDropdown = Comp.Dropdown.makeSingle + , addonRunSubmitted = False } @@ -373,6 +383,12 @@ type Msg | SetNameMsg Comp.SimpleTextInput.Msg | ToggleSelectItem | ItemLinkFormMsg Comp.ItemLinkForm.Msg + | ToggleShowRunAddon + | LoadRunConfigResp (Result Http.Error AddonRunConfigList) + | RunAddonMsg (Comp.Dropdown.Msg AddonRunConfig) + | RunSelectedAddon + | RunAddonResp (Result Http.Error BasicResult) + | SetAddonRunSubmitted Bool type SaveNameState diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/RunAddonForm.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/RunAddonForm.elm new file mode 100644 index 00000000..80705c5e --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/RunAddonForm.elm @@ -0,0 +1,78 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Comp.ItemDetail.RunAddonForm exposing (..) + +import Comp.Basic as B +import Comp.Dropdown +import Comp.ItemDetail.Model exposing (..) +import Comp.MenuBar as MB +import Data.DropdownStyle as DS +import Data.UiSettings exposing (UiSettings) +import Html exposing (Html, div, h3, label, text) +import Html.Attributes exposing (class, classList, title) +import Html.Events exposing (onClick) +import Messages.Comp.ItemDetail.RunAddonForm exposing (Texts) +import Styles as S + + +view : Texts -> UiSettings -> Model -> Html Msg +view texts uiSettings model = + let + viewSettings = + { makeOption = \cfg -> { text = cfg.name, additional = "" } + , placeholder = texts.basics.selectPlaceholder + , labelColor = \_ -> \_ -> "" + , style = DS.mainStyle + } + + runDisabled = + Comp.Dropdown.getSelected model.addonRunConfigDropdown + |> List.isEmpty + in + div + [ classList [ ( "hidden", not model.showRunAddon ) ] + , class "mb-4" + ] + [ h3 [ class S.header3 ] [ text texts.runAddon ] + , div [ class "my-2" ] + [ label [ class S.inputLabel ] + [ text texts.addonRunConfig + ] + , Html.map RunAddonMsg (Comp.Dropdown.view2 viewSettings uiSettings model.addonRunConfigDropdown) + ] + , div [ class "my-2" ] + [ MB.view + { start = + [ MB.CustomElement <| + B.primaryButton + { label = "Run" + , icon = + if model.addonRunSubmitted then + "fa fa-check" + + else + "fa fa-play" + , disabled = runDisabled + , handler = onClick RunSelectedAddon + , attrs = + [ title texts.runAddonTitle + ] + } + , MB.SecondaryButton + { label = texts.basics.cancel + , icon = Just "fa fa-times" + , tagger = ToggleShowRunAddon + , title = "" + } + ] + , end = [] + , rootClasses = "text-sm mt-1" + , sticky = False + } + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm index abc9db46..48871ecb 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/Update.elm @@ -56,6 +56,7 @@ import Comp.PersonForm import Comp.SentMails import Comp.SimpleTextInput import Comp.TagDropdown +import Data.AddonTrigger import Data.CustomFieldChange exposing (CustomFieldChange(..)) import Data.Direction import Data.Environment as Env @@ -75,7 +76,9 @@ import Html5.DragDrop as DD import Http import Page exposing (Page(..)) import Ports +import Process import Set exposing (Set) +import Task import Util.File exposing (makeFileId) import Util.List import Util.Maybe @@ -121,9 +124,77 @@ update inav env msg model = , Cmd.map ItemMailMsg ic , Cmd.map CustomFieldMsg cc , Api.getSentMails env.flags model.item.id SentMailsResp + , Api.addonRunConfigGet env.flags LoadRunConfigResp ] ) + LoadRunConfigResp (Ok list) -> + let + existingItem cfg = + cfg.enabled + && (Data.AddonTrigger.fromList cfg.trigger + |> List.any ((==) Data.AddonTrigger.ExistingItem) + ) + + configs = + List.filter existingItem list.items + + dropdown = + Comp.Dropdown.makeSingleList { options = configs, selected = Nothing } + in + resultModel { model | runConfigs = configs, addonRunConfigDropdown = dropdown } + + RunAddonMsg lm -> + let + ( dd, dc ) = + Comp.Dropdown.update lm model.addonRunConfigDropdown + in + resultModelCmd ( { model | addonRunConfigDropdown = dd }, Cmd.map RunAddonMsg dc ) + + RunSelectedAddon -> + let + configs = + Comp.Dropdown.getSelected model.addonRunConfigDropdown + |> List.map .id + + payload = + { itemId = model.item.id + , additionalItems = [] + , addonRunConfigIds = configs + } + + ( dd, _ ) = + Comp.Dropdown.update (Comp.Dropdown.SetSelection []) model.addonRunConfigDropdown + in + case configs of + [] -> + resultModel model + + _ -> + resultModelCmd + ( { model | addonRunConfigDropdown = dd } + , Api.addonRunExistingItem env.flags payload RunAddonResp + ) + + LoadRunConfigResp (Err _) -> + resultModel model + + RunAddonResp (Ok res) -> + if res.success then + resultModelCmd + ( { model | addonRunSubmitted = True } + , Process.sleep 1200 |> Task.perform (\_ -> SetAddonRunSubmitted False) + ) + + else + resultModel model + + RunAddonResp (Err _) -> + resultModel model + + SetAddonRunSubmitted flag -> + resultModel { model | addonRunSubmitted = flag } + SetItem item -> let res1 = @@ -1638,6 +1709,9 @@ update inav env msg model = , Sub.map ItemLinkFormMsg ils ) + ToggleShowRunAddon -> + resultModel { model | showRunAddon = not model.showRunAddon, mobileItemMenuOpen = False } + --- Helper diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm b/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm index 54a1d3d9..41abc845 100644 --- a/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm +++ b/modules/webapp/src/main/elm/Comp/ItemDetail/View2.elm @@ -22,6 +22,7 @@ import Comp.ItemDetail.Model , isShowQrItem ) import Comp.ItemDetail.Notes +import Comp.ItemDetail.RunAddonForm import Comp.ItemDetail.ShowQrCode import Comp.ItemDetail.SingleAttachment import Comp.ItemLinkForm @@ -177,6 +178,24 @@ menuBar texts inav env model = ] [ Icons.addFilesIcon2 "" ] + , MB.CustomElement <| + a + [ classList + [ ( "bg-gray-200 dark:bg-slate-600", model.showRunAddon ) + , ( "hidden", not env.flags.config.addonsEnabled || List.isEmpty model.runConfigs ) + , ( "hidden md:block", env.flags.config.addonsEnabled && not (List.isEmpty model.runConfigs) ) + ] + , if model.showRunAddon then + title texts.close + + else + title texts.runAddonTitle + , onClick ToggleShowRunAddon + , class S.secondaryBasicButton + , href "#" + ] + [ Icons.addonIcon "" + ] , MB.CustomElement <| a [ classList @@ -248,6 +267,15 @@ menuBar texts inav env model = , onClick AddFilesToggle ] } + , { icon = Icons.addonIcon "" + , label = texts.runAddonLabel + , disabled = False + , attrs = + [ href "#" + , onClick ToggleShowRunAddon + , classList [ ( "hidden", not env.flags.config.addonsEnabled ) ] + ] + } , { icon = Icons.showQrIcon "" , label = texts.showQrCode , disabled = False @@ -402,6 +430,11 @@ itemActions texts flags settings model classes = (S.border ++ " mb-4") model (Comp.ItemDetail.ShowQrCode.Item model.item.id) + , if flags.config.addonsEnabled then + Comp.ItemDetail.RunAddonForm.view texts.runAddonForm settings model + + else + span [ class "hidden" ] [] ] diff --git a/modules/webapp/src/main/elm/Data/AddonTrigger.elm b/modules/webapp/src/main/elm/Data/AddonTrigger.elm new file mode 100644 index 00000000..f790d4e0 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/AddonTrigger.elm @@ -0,0 +1,59 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Data.AddonTrigger exposing (..) + +-- A copy of docspell.addons.AddonTrigger.scala + + +type AddonTrigger + = FinalProcessItem + | FinalReprocessItem + | Scheduled + | ExistingItem + + +all : List AddonTrigger +all = + [ FinalProcessItem + , FinalReprocessItem + , Scheduled + , ExistingItem + ] + + +asString : AddonTrigger -> String +asString t = + case t of + FinalProcessItem -> + "final-process-item" + + FinalReprocessItem -> + "final-reprocess-item" + + Scheduled -> + "scheduled" + + ExistingItem -> + "existing-item" + + +fromString : String -> Maybe AddonTrigger +fromString s = + let + name = + String.toLower s + + x = + List.filter (\e -> asString e == name) all + in + List.head x + + +fromList : List String -> List AddonTrigger +fromList list = + List.filterMap fromString list diff --git a/modules/webapp/src/main/elm/Data/CalEvent.elm b/modules/webapp/src/main/elm/Data/CalEvent.elm index 6453f703..cff62da9 100644 --- a/modules/webapp/src/main/elm/Data/CalEvent.elm +++ b/modules/webapp/src/main/elm/Data/CalEvent.elm @@ -8,6 +8,7 @@ module Data.CalEvent exposing ( CalEvent , everyMonth + , everyMonthTz , fromEvent , makeEvent ) @@ -29,7 +30,12 @@ type alias CalEvent = everyMonth : CalEvent everyMonth = - CalEvent Nothing "*" "*" "01" "00" "00" Data.TimeZone.utc + everyMonthTz Data.TimeZone.utc + + +everyMonthTz : TimeZone -> CalEvent +everyMonthTz tz = + CalEvent Nothing "*" "*" "01" "00" "00" tz makeEvent : CalEvent -> String diff --git a/modules/webapp/src/main/elm/Data/Flags.elm b/modules/webapp/src/main/elm/Data/Flags.elm index f91aba31..614b452a 100644 --- a/modules/webapp/src/main/elm/Data/Flags.elm +++ b/modules/webapp/src/main/elm/Data/Flags.elm @@ -38,6 +38,7 @@ type alias Config = , downloadAllMaxFiles : Int , downloadAllMaxSize : Int , openIdAuth : List OpenIdAuth + , addonsEnabled : Bool } diff --git a/modules/webapp/src/main/elm/Data/Icons.elm b/modules/webapp/src/main/elm/Data/Icons.elm index 236390a7..c50cebb5 100644 --- a/modules/webapp/src/main/elm/Data/Icons.elm +++ b/modules/webapp/src/main/elm/Data/Icons.elm @@ -8,6 +8,8 @@ module Data.Icons exposing ( addFiles2 , addFilesIcon2 + , addonIcon + , addonRunConfigIcon , concerned , concerned2 , concernedIcon @@ -76,7 +78,7 @@ module Data.Icons exposing ) import Data.CustomFieldType exposing (CustomFieldType) -import Html exposing (Html, i, img) +import Html exposing (Html, div, i, img, span) import Html.Attributes exposing (class, src) import Svg import Svg.Attributes as SA @@ -265,6 +267,24 @@ customFieldIcon2 classes = i [ class (customField2 ++ " " ++ classes) ] [] +addon : String +addon = + "fa fa-puzzle-piece" + + +addonIcon : String -> Html msg +addonIcon classes = + i [ class (addon ++ " " ++ classes) ] [] + + +addonRunConfigIcon : String -> Html msg +addonRunConfigIcon classes = + div [ class (classes ++ " inline-block relative margin-auto leading-8") ] + [ i [ class "fa fa-puzzle-piece" ] [] + , i [ class "fa fa-play font-bold absolute text-xs -right-2 top-0" ] [] + ] + + search : String search = "fa fa-search" diff --git a/modules/webapp/src/main/elm/Data/ServerEvent.elm b/modules/webapp/src/main/elm/Data/ServerEvent.elm index 194ba0cc..57320996 100644 --- a/modules/webapp/src/main/elm/Data/ServerEvent.elm +++ b/modules/webapp/src/main/elm/Data/ServerEvent.elm @@ -5,15 +5,34 @@ -} -module Data.ServerEvent exposing (ServerEvent(..), decode) +module Data.ServerEvent exposing (AddonInfo, ServerEvent(..), decode) import Json.Decode as D +import Json.Decode.Pipeline as P type ServerEvent = JobSubmitted String | JobDone String | JobsWaiting Int + | AddonInstalled AddonInfo + + +type alias AddonInfo = + { success : Bool + , addonId : Maybe String + , addonUrl : Maybe String + , message : String + } + + +addonInfoDecoder : D.Decoder AddonInfo +addonInfoDecoder = + D.succeed AddonInfo + |> P.required "success" D.bool + |> P.optional "addonId" (D.maybe D.string) Nothing + |> P.optional "addonUrl" (D.maybe D.string) Nothing + |> P.required "message" D.string decoder : D.Decoder ServerEvent @@ -43,5 +62,9 @@ decodeTag tag = D.field "content" D.int |> D.map JobsWaiting + "addon-installed" -> + D.field "content" addonInfoDecoder + |> D.map AddonInstalled + _ -> D.fail ("Unknown tag: " ++ tag) diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveForm.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveForm.elm new file mode 100644 index 00000000..3bac22c4 --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveForm.elm @@ -0,0 +1,54 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonArchiveForm exposing + ( Texts + , de + , fr + , gb + ) + +import Messages.Basics + + +type alias Texts = + { basics : Messages.Basics.Texts + , addonUrl : String + , addonUrlPlaceholder : String + , installInfoText : String + } + + +gb : Texts +gb = + { basics = Messages.Basics.gb + , addonUrl = "Addon URL" + , addonUrlPlaceholder = "e.g. https://github.com/some-user/project/refs/tags/1.0.zip" + , installInfoText = "Only urls to remote addon zip files are supported." + } + + +de : Texts +de = + { basics = Messages.Basics.de + , addonUrl = "Addon URL" + , addonUrlPlaceholder = "z.B. https://github.com/some-user/project/refs/tags/1.0.zip" + , installInfoText = "Nur URLs to externen zip Dateien werden unterstützt." + } + + + +-- TODO: translate-fr + + +fr : Texts +fr = + { basics = Messages.Basics.fr + , addonUrl = "Addon URL" + , addonUrlPlaceholder = "p.e. https://github.com/some-user/project/refs/tags/1.0.zip" + , installInfoText = "Only urls to remote addon zip files are supported." + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveManage.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveManage.elm new file mode 100644 index 00000000..5cc6b135 --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveManage.elm @@ -0,0 +1,86 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonArchiveManage exposing + ( Texts + , de + , fr + , gb + ) + +import Http +import Messages.Basics +import Messages.Comp.AddonArchiveForm +import Messages.Comp.AddonArchiveTable +import Messages.Comp.HttpError + + +type alias Texts = + { basics : Messages.Basics.Texts + , addonArchiveTable : Messages.Comp.AddonArchiveTable.Texts + , addonArchiveForm : Messages.Comp.AddonArchiveForm.Texts + , httpError : Http.Error -> String + , newAddonArchive : String + , reallyDeleteAddonArchive : String + , createNewAddonArchive : String + , deleteThisAddonArchive : String + , correctFormErrors : String + , installNow : String + , updateNow : String + , description : String + } + + +gb : Texts +gb = + { basics = Messages.Basics.gb + , addonArchiveTable = Messages.Comp.AddonArchiveTable.gb + , addonArchiveForm = Messages.Comp.AddonArchiveForm.gb + , httpError = Messages.Comp.HttpError.gb + , newAddonArchive = "New Addon" + , reallyDeleteAddonArchive = "Really delete this Addon?" + , createNewAddonArchive = "Install new Addon" + , deleteThisAddonArchive = "Delete this Addon" + , correctFormErrors = "Please correct the errors in the form." + , installNow = "Install Addon" + , updateNow = "Update Addon" + , description = "Description" + } + + +de : Texts +de = + { basics = Messages.Basics.de + , addonArchiveTable = Messages.Comp.AddonArchiveTable.de + , addonArchiveForm = Messages.Comp.AddonArchiveForm.de + , httpError = Messages.Comp.HttpError.de + , newAddonArchive = "Neues Addon" + , reallyDeleteAddonArchive = "Dieses Addon wirklich entfernen?" + , createNewAddonArchive = "Neues Addon installieren" + , deleteThisAddonArchive = "Addon löschen" + , correctFormErrors = "Bitte korrigiere die Fehler im Formular." + , installNow = "Addon Installieren" + , updateNow = "Addon aktualisieren" + , description = "Beschreibung" + } + + +fr : Texts +fr = + { basics = Messages.Basics.fr + , addonArchiveTable = Messages.Comp.AddonArchiveTable.fr + , addonArchiveForm = Messages.Comp.AddonArchiveForm.fr + , httpError = Messages.Comp.HttpError.fr + , newAddonArchive = "Nouveau favori" + , reallyDeleteAddonArchive = "Confirmer la suppression de ce favori ?" + , createNewAddonArchive = "Créer un nouveau favori" + , deleteThisAddonArchive = "Supprimer ce favori" + , correctFormErrors = "Veuillez corriger les erreurs du formulaire" + , installNow = "Installation de l'addon" + , updateNow = "Actualiser l'addon" + , description = "Description" + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveTable.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveTable.elm new file mode 100644 index 00000000..47f95eee --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonArchiveTable.elm @@ -0,0 +1,42 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonArchiveTable exposing + ( Texts + , de + , fr + , gb + ) + +import Messages.Basics + + +type alias Texts = + { basics : Messages.Basics.Texts + , version : String + } + + +gb : Texts +gb = + { basics = Messages.Basics.gb + , version = "Version" + } + + +de : Texts +de = + { basics = Messages.Basics.de + , version = "Version" + } + + +fr : Texts +fr = + { basics = Messages.Basics.fr + , version = "Version" + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigForm.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigForm.elm new file mode 100644 index 00000000..181239f9 --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigForm.elm @@ -0,0 +1,108 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonRunConfigForm exposing + ( Texts + , de + , fr + , gb + ) + +import Data.TimeZone exposing (TimeZone) +import Messages.Basics +import Messages.Comp.CalEventInput + + +type alias Texts = + { basics : Messages.Basics.Texts + , calEventInput : Messages.Comp.CalEventInput.Texts + , enableDisable : String + , chooseName : String + , impersonateUser : String + , triggerRun : String + , schedule : String + , addons : String + , includedAddons : String + , add : String + , readMore : String + , readLess : String + , arguments : String + , update : String + , argumentsUpdated : String + , configureTitle : String + , configureLabel : String + } + + +gb : TimeZone -> Texts +gb tz = + { basics = Messages.Basics.gb + , calEventInput = Messages.Comp.CalEventInput.gb tz + , enableDisable = "Enable or disable this run configuration." + , chooseName = "Choose a name…" + , impersonateUser = "Run on behalf of user" + , triggerRun = "Trigger Run" + , schedule = "Schedule" + , addons = "Addons" + , includedAddons = "Included addons" + , add = "Add" + , readMore = "Read more" + , readLess = "Read less" + , arguments = "Arguments" + , update = "Update" + , argumentsUpdated = "Arguments updated" + , configureTitle = "Configure this addon" + , configureLabel = "Configure" + } + + +de : TimeZone -> Texts +de tz = + { basics = Messages.Basics.de + , calEventInput = Messages.Comp.CalEventInput.de tz + , enableDisable = "Konfiguration aktivieren oder deaktivieren" + , chooseName = "Name der Konfiguration…" + , impersonateUser = "Als Benutzer ausführen" + , triggerRun = "Auslöser" + , schedule = "Zeitplan" + , addons = "Addons" + , includedAddons = "Gewählte Addons" + , add = "Hinzufügen" + , readMore = "Mehr" + , readLess = "Weniger" + , arguments = "Argumente" + , update = "Aktualisieren" + , argumentsUpdated = "Argumente aktualisiert" + , configureTitle = "Konfiguriere dieses Addon" + , configureLabel = "Konfigurieren" + } + + + +-- TODO: translate-fr + + +fr : TimeZone -> Texts +fr tz = + { basics = Messages.Basics.fr + , calEventInput = Messages.Comp.CalEventInput.fr tz + , enableDisable = "Activer ou désactiver cette tâche." + , chooseName = "Choose a name…" + , impersonateUser = "Impersonate user" + , triggerRun = "Trigger Run" + , schedule = "Programmation" + , addons = "Addons" + , includedAddons = "Included addons" + , add = "Ajouter" + , readMore = "Read more" + , readLess = "Read less" + , arguments = "Arguments" + , update = "Update" + , argumentsUpdated = "Arguments updated" + , configureTitle = "Configure this addon" + , configureLabel = "Configure" + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigManage.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigManage.elm new file mode 100644 index 00000000..b50a17fc --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigManage.elm @@ -0,0 +1,79 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonRunConfigManage exposing + ( Texts + , de + , fr + , gb + ) + +import Data.TimeZone exposing (TimeZone) +import Http +import Messages.Basics +import Messages.Comp.AddonRunConfigForm +import Messages.Comp.AddonRunConfigTable +import Messages.Comp.HttpError + + +type alias Texts = + { basics : Messages.Basics.Texts + , addonArchiveTable : Messages.Comp.AddonRunConfigTable.Texts + , addonArchiveForm : Messages.Comp.AddonRunConfigForm.Texts + , httpError : Http.Error -> String + , newAddonRunConfig : String + , reallyDeleteAddonRunConfig : String + , createNewAddonRunConfig : String + , deleteThisAddonRunConfig : String + , correctFormErrors : String + } + + +gb : TimeZone -> Texts +gb tz = + { basics = Messages.Basics.gb + , addonArchiveTable = Messages.Comp.AddonRunConfigTable.gb + , addonArchiveForm = Messages.Comp.AddonRunConfigForm.gb tz + , httpError = Messages.Comp.HttpError.gb + , newAddonRunConfig = "New" + , reallyDeleteAddonRunConfig = "Really delete this run config?" + , createNewAddonRunConfig = "Create a new run configuration" + , deleteThisAddonRunConfig = "Delete this run configuration" + , correctFormErrors = "Please correct the errors in the form." + } + + +de : TimeZone -> Texts +de tz = + { basics = Messages.Basics.de + , addonArchiveTable = Messages.Comp.AddonRunConfigTable.de + , addonArchiveForm = Messages.Comp.AddonRunConfigForm.de tz + , httpError = Messages.Comp.HttpError.de + , newAddonRunConfig = "Neu" + , reallyDeleteAddonRunConfig = "Dieses Konfiguration wirklich entfernen?" + , createNewAddonRunConfig = "Neue Run-Konfiguration erstellen" + , deleteThisAddonRunConfig = "Run-Konfiguration löschen" + , correctFormErrors = "Bitte korrigiere die Fehler im Formular." + } + + + +--- TODO translate-fr + + +fr : TimeZone -> Texts +fr tz = + { basics = Messages.Basics.fr + , addonArchiveTable = Messages.Comp.AddonRunConfigTable.fr + , addonArchiveForm = Messages.Comp.AddonRunConfigForm.fr tz + , httpError = Messages.Comp.HttpError.fr + , newAddonRunConfig = "Nouveau favori" + , reallyDeleteAddonRunConfig = "Confirmer la suppression de ce favori ?" + , createNewAddonRunConfig = "Créer un nouveau favori" + , deleteThisAddonRunConfig = "Supprimer ce favori" + , correctFormErrors = "Veuillez corriger les erreurs du formulaire" + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigTable.elm b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigTable.elm new file mode 100644 index 00000000..02c4b417 --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/AddonRunConfigTable.elm @@ -0,0 +1,50 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.AddonRunConfigTable exposing + ( Texts + , de + , fr + , gb + ) + +import Messages.Basics + + +type alias Texts = + { basics : Messages.Basics.Texts + , enabled : String + , trigger : String + } + + +gb : Texts +gb = + { basics = Messages.Basics.gb + , enabled = "Enabled" + , trigger = "Triggered" + } + + +de : Texts +de = + { basics = Messages.Basics.de + , enabled = "Aktive" + , trigger = "Auslöser" + } + + + +-- TODO translate-fr + + +fr : Texts +fr = + { basics = Messages.Basics.fr + , enabled = "Enabled" + , trigger = "Triggered" + } diff --git a/modules/webapp/src/main/elm/Messages/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Messages/Comp/ItemDetail.elm index 3ea89606..fca6e904 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/ItemDetail.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/ItemDetail.elm @@ -20,6 +20,7 @@ import Messages.Comp.ItemDetail.AddFilesForm import Messages.Comp.ItemDetail.ConfirmModal import Messages.Comp.ItemDetail.ItemInfoHeader import Messages.Comp.ItemDetail.Notes +import Messages.Comp.ItemDetail.RunAddonForm import Messages.Comp.ItemDetail.SingleAttachment import Messages.Comp.ItemLinkForm import Messages.Comp.ItemMail @@ -38,6 +39,7 @@ type alias Texts = , detailEdit : Messages.Comp.DetailEdit.Texts , confirmModal : Messages.Comp.ItemDetail.ConfirmModal.Texts , itemLinkForm : Messages.Comp.ItemLinkForm.Texts + , runAddonForm : Messages.Comp.ItemDetail.RunAddonForm.Texts , httpError : Http.Error -> String , key : String , backToSearchResults : String @@ -64,6 +66,8 @@ type alias Texts = , selectItem : String , deselectItem : String , relatedItems : String + , runAddonLabel : String + , runAddonTitle : String } @@ -78,6 +82,7 @@ gb tz = , detailEdit = Messages.Comp.DetailEdit.gb , confirmModal = Messages.Comp.ItemDetail.ConfirmModal.gb , itemLinkForm = Messages.Comp.ItemLinkForm.gb tz + , runAddonForm = Messages.Comp.ItemDetail.RunAddonForm.gb , httpError = Messages.Comp.HttpError.gb , key = "Key" , backToSearchResults = "Back to search results" @@ -104,6 +109,8 @@ gb tz = , selectItem = "Select this item" , deselectItem = "Deselect this item" , relatedItems = "Linked items" + , runAddonLabel = "Run addon" + , runAddonTitle = "Run an addon on this item" } @@ -118,6 +125,7 @@ de tz = , detailEdit = Messages.Comp.DetailEdit.de , confirmModal = Messages.Comp.ItemDetail.ConfirmModal.de , itemLinkForm = Messages.Comp.ItemLinkForm.de tz + , runAddonForm = Messages.Comp.ItemDetail.RunAddonForm.de , httpError = Messages.Comp.HttpError.de , key = "Taste" , backToSearchResults = "Zurück zur Suche" @@ -144,6 +152,8 @@ de tz = , selectItem = "Zur Auswahl hinzufügen" , deselectItem = "Aus Auswahl entfernen" , relatedItems = "Verknüpfte Dokumente" + , runAddonLabel = "Addon ausführen" + , runAddonTitle = "Addons für dieses Dokument ausführen" } @@ -158,6 +168,7 @@ fr tz = , detailEdit = Messages.Comp.DetailEdit.fr , confirmModal = Messages.Comp.ItemDetail.ConfirmModal.fr , itemLinkForm = Messages.Comp.ItemLinkForm.fr tz + , runAddonForm = Messages.Comp.ItemDetail.RunAddonForm.fr , httpError = Messages.Comp.HttpError.fr , key = "Clé" , backToSearchResults = "Retour aux résultat de recherche" @@ -184,4 +195,10 @@ fr tz = , selectItem = "Sélectionner ce document" , deselectItem = "Désélectionner ce document" , relatedItems = "Documents associés" + , runAddonLabel = "Run addon" + , runAddonTitle = "Run an addon on this item" } + + + +-- TODO translate-fr diff --git a/modules/webapp/src/main/elm/Messages/Comp/ItemDetail/RunAddonForm.elm b/modules/webapp/src/main/elm/Messages/Comp/ItemDetail/RunAddonForm.elm new file mode 100644 index 00000000..9901c55b --- /dev/null +++ b/modules/webapp/src/main/elm/Messages/Comp/ItemDetail/RunAddonForm.elm @@ -0,0 +1,49 @@ +{- + Copyright 2020 Eike K. & Contributors + + SPDX-License-Identifier: AGPL-3.0-or-later +-} + + +module Messages.Comp.ItemDetail.RunAddonForm exposing (Texts, de, fr, gb) + +import Messages.Basics + + +type alias Texts = + { basics : Messages.Basics.Texts + , runAddon : String + , addonRunConfig : String + , runAddonTitle : String + } + + +gb : Texts +gb = + { basics = Messages.Basics.gb + , runAddon = "Run an addon" + , addonRunConfig = "Addon run configuration" + , runAddonTitle = "Run the selected addon on this item." + } + + +de : Texts +de = + { basics = Messages.Basics.de + , runAddon = "Addon ausführen" + , addonRunConfig = "Addon Konfiguration" + , runAddonTitle = "Run the selected addon on this item." + } + + + +-- TODO: translate-fr + + +fr : Texts +fr = + { basics = Messages.Basics.fr + , runAddon = "Run an addon" + , addonRunConfig = "Addon run configuration" + , runAddonTitle = "Run the selected addon on this item." + } diff --git a/modules/webapp/src/main/elm/Messages/Page/ManageData.elm b/modules/webapp/src/main/elm/Messages/Page/ManageData.elm index 5760f0f6..6ce47932 100644 --- a/modules/webapp/src/main/elm/Messages/Page/ManageData.elm +++ b/modules/webapp/src/main/elm/Messages/Page/ManageData.elm @@ -14,6 +14,8 @@ module Messages.Page.ManageData exposing import Data.TimeZone exposing (TimeZone) import Messages.Basics +import Messages.Comp.AddonArchiveManage +import Messages.Comp.AddonRunConfigManage import Messages.Comp.BookmarkManage import Messages.Comp.CustomFieldManage import Messages.Comp.EquipmentManage @@ -32,8 +34,12 @@ type alias Texts = , folderManage : Messages.Comp.FolderManage.Texts , customFieldManage : Messages.Comp.CustomFieldManage.Texts , bookmarkManage : Messages.Comp.BookmarkManage.Texts + , addonArchiveManage : Messages.Comp.AddonArchiveManage.Texts + , addonRunConfigManage : Messages.Comp.AddonRunConfigManage.Texts , manageData : String , bookmarks : String + , addonArchives : String + , addonRunConfigs : String } @@ -47,8 +53,12 @@ gb tz = , folderManage = Messages.Comp.FolderManage.gb tz , customFieldManage = Messages.Comp.CustomFieldManage.gb tz , bookmarkManage = Messages.Comp.BookmarkManage.gb + , addonArchiveManage = Messages.Comp.AddonArchiveManage.gb + , addonRunConfigManage = Messages.Comp.AddonRunConfigManage.gb tz , manageData = "Manage Data" , bookmarks = "Bookmarks" + , addonArchives = "Addons" + , addonRunConfigs = "Addon Run Configurations" } @@ -62,8 +72,12 @@ de tz = , folderManage = Messages.Comp.FolderManage.de tz , customFieldManage = Messages.Comp.CustomFieldManage.de tz , bookmarkManage = Messages.Comp.BookmarkManage.de + , addonArchiveManage = Messages.Comp.AddonArchiveManage.de + , addonRunConfigManage = Messages.Comp.AddonRunConfigManage.de tz , manageData = "Daten verwalten" , bookmarks = "Bookmarks" + , addonArchives = "Addons" + , addonRunConfigs = "Addon Run Configurations" } @@ -77,6 +91,10 @@ fr tz = , folderManage = Messages.Comp.FolderManage.fr tz , customFieldManage = Messages.Comp.CustomFieldManage.fr tz , bookmarkManage = Messages.Comp.BookmarkManage.fr + , addonArchiveManage = Messages.Comp.AddonArchiveManage.fr + , addonRunConfigManage = Messages.Comp.AddonRunConfigManage.fr tz , manageData = "Gestion des métadonnées" , bookmarks = "Favoris" + , addonArchives = "Addons" + , addonRunConfigs = "Addon Run Configurations" } diff --git a/modules/webapp/src/main/elm/Page/ManageData/Data.elm b/modules/webapp/src/main/elm/Page/ManageData/Data.elm index f56b8702..e0a0fd45 100644 --- a/modules/webapp/src/main/elm/Page/ManageData/Data.elm +++ b/modules/webapp/src/main/elm/Page/ManageData/Data.elm @@ -12,6 +12,8 @@ module Page.ManageData.Data exposing , init ) +import Comp.AddonArchiveManage +import Comp.AddonRunConfigManage import Comp.BookmarkManage import Comp.CustomFieldManage import Comp.EquipmentManage @@ -31,6 +33,8 @@ type alias Model = , folderManageModel : Comp.FolderManage.Model , fieldManageModel : Comp.CustomFieldManage.Model , bookmarkModel : Comp.BookmarkManage.Model + , addonArchiveModel : Comp.AddonArchiveManage.Model + , addonRunConfigModel : Comp.AddonRunConfigManage.Model } @@ -42,6 +46,12 @@ init flags = ( bm, bc ) = Comp.BookmarkManage.init flags + + ( aam, aac ) = + Comp.AddonArchiveManage.init flags + + ( arm, arc ) = + Comp.AddonRunConfigManage.init flags in ( { currentTab = Just TagTab , tagManageModel = m2 @@ -51,10 +61,14 @@ init flags = , folderManageModel = Comp.FolderManage.empty , fieldManageModel = Comp.CustomFieldManage.empty , bookmarkModel = bm + , addonArchiveModel = aam + , addonRunConfigModel = arm } , Cmd.batch [ Cmd.map TagManageMsg c2 , Cmd.map BookmarkMsg bc + , Cmd.map AddonArchiveMsg aac + , Cmd.map AddonRunConfigMsg arc ] ) @@ -67,6 +81,8 @@ type Tab | FolderTab | CustomFieldTab | BookmarkTab + | AddonArchiveTab + | AddonRunConfigTab type Msg @@ -78,3 +94,5 @@ type Msg | FolderMsg Comp.FolderManage.Msg | CustomFieldMsg Comp.CustomFieldManage.Msg | BookmarkMsg Comp.BookmarkManage.Msg + | AddonArchiveMsg Comp.AddonArchiveManage.Msg + | AddonRunConfigMsg Comp.AddonRunConfigManage.Msg diff --git a/modules/webapp/src/main/elm/Page/ManageData/Update.elm b/modules/webapp/src/main/elm/Page/ManageData/Update.elm index b2c61dd3..981f1b38 100644 --- a/modules/webapp/src/main/elm/Page/ManageData/Update.elm +++ b/modules/webapp/src/main/elm/Page/ManageData/Update.elm @@ -7,6 +7,8 @@ module Page.ManageData.Update exposing (update) +import Comp.AddonArchiveManage +import Comp.AddonRunConfigManage import Comp.BookmarkManage import Comp.CustomFieldManage import Comp.EquipmentManage @@ -15,11 +17,12 @@ import Comp.OrgManage import Comp.PersonManage import Comp.TagManage import Data.Flags exposing (Flags) +import Data.UiSettings exposing (UiSettings) import Page.ManageData.Data exposing (..) -update : Flags -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg ) -update flags msg model = +update : Flags -> UiSettings -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg ) +update flags uiSettings msg model = case msg of SetTab t -> let @@ -28,16 +31,16 @@ update flags msg model = in case t of TagTab -> - update flags (TagManageMsg Comp.TagManage.LoadTags) m + update flags uiSettings (TagManageMsg Comp.TagManage.LoadTags) m EquipTab -> - update flags (EquipManageMsg Comp.EquipmentManage.LoadEquipments) m + update flags uiSettings (EquipManageMsg Comp.EquipmentManage.LoadEquipments) m OrgTab -> - update flags (OrgManageMsg Comp.OrgManage.LoadOrgs) m + update flags uiSettings (OrgManageMsg Comp.OrgManage.LoadOrgs) m PersonTab -> - update flags (PersonManageMsg Comp.PersonManage.LoadPersons) m + update flags uiSettings (PersonManageMsg Comp.PersonManage.LoadPersons) m FolderTab -> let @@ -60,6 +63,20 @@ update flags msg model = in ( { m | bookmarkModel = bm }, Cmd.map BookmarkMsg bc, Sub.none ) + AddonArchiveTab -> + let + ( aam, aac ) = + Comp.AddonArchiveManage.init flags + in + ( { m | addonArchiveModel = aam }, Cmd.map AddonArchiveMsg aac, Sub.none ) + + AddonRunConfigTab -> + let + ( arm, arc ) = + Comp.AddonRunConfigManage.init flags + in + ( { m | addonRunConfigModel = arm }, Cmd.map AddonRunConfigMsg arc, Sub.none ) + TagManageMsg m -> let ( m2, c2 ) = @@ -117,3 +134,23 @@ update flags msg model = , Cmd.map BookmarkMsg c2 , Sub.map BookmarkMsg s2 ) + + AddonArchiveMsg lm -> + let + ( aam, aac, aas ) = + Comp.AddonArchiveManage.update flags lm model.addonArchiveModel + in + ( { model | addonArchiveModel = aam } + , Cmd.map AddonArchiveMsg aac + , Sub.map AddonArchiveMsg aas + ) + + AddonRunConfigMsg lm -> + let + ( arm, arc, ars ) = + Comp.AddonRunConfigManage.update flags uiSettings.timeZone lm model.addonRunConfigModel + in + ( { model | addonRunConfigModel = arm } + , Cmd.map AddonRunConfigMsg arc + , Sub.map AddonRunConfigMsg ars + ) diff --git a/modules/webapp/src/main/elm/Page/ManageData/View2.elm b/modules/webapp/src/main/elm/Page/ManageData/View2.elm index 2354a7c3..53f814dc 100644 --- a/modules/webapp/src/main/elm/Page/ManageData/View2.elm +++ b/modules/webapp/src/main/elm/Page/ManageData/View2.elm @@ -7,6 +7,8 @@ module Page.ManageData.View2 exposing (viewContent, viewSidebar) +import Comp.AddonArchiveManage +import Comp.AddonRunConfigManage import Comp.BookmarkManage import Comp.CustomFieldManage import Comp.EquipmentManage @@ -27,7 +29,7 @@ import Styles as S viewSidebar : Texts -> Bool -> Flags -> UiSettings -> Model -> Html Msg -viewSidebar texts visible _ settings model = +viewSidebar texts visible flags settings model = div [ id "sidebar" , class S.sidebar @@ -134,6 +136,32 @@ viewSidebar texts visible _ settings model = [ text texts.bookmarks ] ] + , a + [ href "#" + , onClick (SetTab AddonArchiveTab) + , menuEntryActive model AddonArchiveTab + , class S.sidebarLink + , classList [ ( "hidden", not flags.config.addonsEnabled ) ] + ] + [ Icons.addonIcon "" + , span + [ class "ml-3" ] + [ text texts.addonArchives + ] + ] + , a + [ href "#" + , onClick (SetTab AddonRunConfigTab) + , menuEntryActive model AddonRunConfigTab + , class S.sidebarLink + , classList [ ( "hidden", not flags.config.addonsEnabled ) ] + ] + [ Icons.addonRunConfigIcon "" + , span + [ class "ml-3" ] + [ text texts.addonRunConfigs + ] + ] ] ] @@ -166,6 +194,20 @@ viewContent texts flags settings model = Just BookmarkTab -> viewBookmarks texts flags settings model + Just AddonArchiveTab -> + if flags.config.addonsEnabled then + viewAddonArchives texts flags settings model + + else + [] + + Just AddonRunConfigTab -> + if flags.config.addonsEnabled then + viewAddonRunConfigs texts flags settings model + + else + [] + Nothing -> [] ) @@ -306,3 +348,33 @@ viewBookmarks texts flags settings model = ] , Html.map BookmarkMsg (Comp.BookmarkManage.view texts.bookmarkManage settings flags model.bookmarkModel) ] + + +viewAddonArchives : Texts -> Flags -> UiSettings -> Model -> List (Html Msg) +viewAddonArchives texts flags settings model = + [ h2 + [ class S.header1 + , class "inline-flex items-center" + ] + [ Icons.addonIcon "" + , div [ class "ml-2" ] + [ text texts.addonArchives + ] + ] + , Html.map AddonArchiveMsg (Comp.AddonArchiveManage.view texts.addonArchiveManage settings flags model.addonArchiveModel) + ] + + +viewAddonRunConfigs : Texts -> Flags -> UiSettings -> Model -> List (Html Msg) +viewAddonRunConfigs texts flags settings model = + [ h2 + [ class S.header1 + , class "inline-flex items-center" + ] + [ Icons.addonRunConfigIcon "mr-4" + , div [ class "ml-2" ] + [ text texts.addonRunConfigs + ] + ] + , Html.map AddonRunConfigMsg (Comp.AddonRunConfigManage.view texts.addonRunConfigManage settings flags model.addonRunConfigModel) + ] diff --git a/modules/webapp/src/main/elm/Styles.elm b/modules/webapp/src/main/elm/Styles.elm index 48a866c4..a9a9304b 100644 --- a/modules/webapp/src/main/elm/Styles.elm +++ b/modules/webapp/src/main/elm/Styles.elm @@ -35,6 +35,11 @@ sidebarLink = " mb-2 px-4 py-3 flex flex-row hover:bg-blue-100 dark:hover:bg-slate-600 hover:font-bold rounded rounded-lg items-center " +successText : String +successText = + " text-green-600 dark:text-lime-500 " + + successMessage : String successMessage = " border border-green-600 bg-green-50 text-green-600 dark:border-lime-800 dark:bg-lime-300 dark:text-lime-800 px-4 py-2 rounded " @@ -281,7 +286,7 @@ link = inputErrorBorder : String inputErrorBorder = - " border-red-600 dark:border-orange-600 " + " ring dark:ring-0 ring-red-600 dark:border-orange-600 " inputLabel : String diff --git a/modules/webapp/src/main/elm/Util/List.elm b/modules/webapp/src/main/elm/Util/List.elm index 0c532bcb..ce7d1250 100644 --- a/modules/webapp/src/main/elm/Util/List.elm +++ b/modules/webapp/src/main/elm/Util/List.elm @@ -14,12 +14,42 @@ module Util.List exposing , findNext , findPrev , get + , removeByIndex + , replaceByIndex , sliding ) import Html.Attributes exposing (list) +removeByIndex : Int -> List a -> List a +removeByIndex index list = + List.indexedMap + (\idx -> + \e -> + if idx == index then + Nothing + + else + Just e + ) + list + |> List.filterMap identity + + +replaceByIndex : Int -> a -> List a -> List a +replaceByIndex index element list = + let + repl idx e = + if idx == index then + element + + else + e + in + List.indexedMap repl list + + changePosition : Int -> Int -> List a -> List a changePosition source target list = let diff --git a/modules/webapp/src/main/elm/Util/String.elm b/modules/webapp/src/main/elm/Util/String.elm index 54cfa9ea..a5972924 100644 --- a/modules/webapp/src/main/elm/Util/String.elm +++ b/modules/webapp/src/main/elm/Util/String.elm @@ -9,6 +9,7 @@ module Util.String exposing ( appendIfAbsent , crazyEncode , ellipsis + , firstSentenceOrMax , isBlank , isNothingOrBlank , underscoreToSpace @@ -16,7 +17,6 @@ module Util.String exposing ) import Base64 -import Html exposing (strong) crazyEncode : String -> String @@ -45,6 +45,26 @@ ellipsis len str = String.left (len - 1) str ++ "…" +firstSentenceOrMax : Int -> String -> Maybe String +firstSentenceOrMax maxLen str = + let + idx = + String.indexes "." str + |> List.head + |> Maybe.map ((+) 2) + |> Maybe.map (min maxLen) + |> Maybe.withDefault maxLen + + len = + String.length str + in + if len <= maxLen then + Nothing + + else + Just <| String.left (idx - 1) str ++ "…" + + withDefault : String -> String -> String withDefault default str = if str == "" then diff --git a/modules/webapp/src/main/styles/custom-components.css b/modules/webapp/src/main/styles/custom-components.css index 5daaf40b..afb3edca 100644 --- a/modules/webapp/src/main/styles/custom-components.css +++ b/modules/webapp/src/main/styles/custom-components.css @@ -96,4 +96,8 @@ .markdown-preview a { @apply text-blue-400 hover:text-blue-500 dark:text-sky-200 dark:hover:text-sky-100 cursor-pointer; } + + .markdown-preview pre { + @apply font-mono px-2 py-2 text-sm border dark:border-slate-600 rounded my-2; + } }