TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Experiment with addons

Browse Source 
Addons allow to execute external programs in some context inside
docspell. Currently it is possible to run them after processing files.
Addons are provided by URLs to zip files.
This commit is contained in:
eikek
2022-04-22 14:07:28 +02:00
parent e04a76faa4
commit 7fdd78ad06
166 changed files with 8181 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
modules/restserver/src/main/resources/reference.conf
View File

@ -462,5 +462,39 @@ docspell.server {
}
}
}
addons = {
enabled = false
# Whether installing addons requiring network should be allowed
# or not.
allow-impure = true
# Define patterns of urls that are allowed to install addons
# from.
#
# A pattern is compared against an URL by comparing three parts
# of an URL via globs: scheme, host and path.
#
# You can use '*' (0 or more) and '?' (one) as wildcards in each
# part. For example:
#
# https://*.mydomain.com/projects/*
# *s://gitea.mydomain/*
#
# A hostname is separated by dots and the path by a slash. A '*'
# in a pattern means to match one or more characters. The path
# pattern is always matching the given prefix. So /a/b/* matches
# /a/b/c and /a/b/c/d and all other sub-paths.
#
# Multiple patterns can be defined va a comma separated string
# or as an array. An empty string matches no URL, while the
# special pattern '*' all by itself means to match every URL.
allowed-urls = "*"
# Same as `allowed-urls` but a match here means do deny addons
# from this url.
denied-urls = ""
}
}
}

15
modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala
View File

@ -152,7 +152,9 @@ final class RestAppImpl[F[_]: Async](
"clientSettings" -> ClientSettingsRoutes(backend, token),
"notification" -> NotificationRoutes(config, backend, token),
"querybookmark" -> BookmarkRoutes(backend, token),
"downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token)
"downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token),
"addonrunconfig" -> AddonRunConfigRoutes(backend, token),
"addon" -> AddonRoutes(config, wsTopic, backend, token)
)
}
@ -181,7 +183,16 @@ object RestAppImpl {
.withEventSink(notificationMod)
.build
backend <- BackendApp
.create[F](store, javaEmil, ftsClient, pubSubT, schedulerMod, notificationMod)
.create[F](
cfg.backend,
store,
javaEmil,
httpClient,
ftsClient,
pubSubT,
schedulerMod,
notificationMod
)
app = new RestAppImpl[F](
cfg,

10
modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
View File

@ -14,6 +14,7 @@ import fs2.Stream
import fs2.concurrent.Topic
import docspell.backend.msg.Topics
import docspell.backend.ops.ONode
import docspell.common._
import docspell.pubsub.naive.NaivePubSub
import docspell.restserver.http4s.InternalHeader
@ -91,6 +92,15 @@ object RestServer {
store,
httpClient
)(Topics.all.map(_.topic))
nodes <- ONode(store)
_ <- nodes.withRegistered(
cfg.appId,
NodeType.Restserver,
cfg.baseUrl,
cfg.auth.serverSecret.some
)
restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic)
} yield (restApp, pubSub, setting)

70
modules/restserver/src/main/scala/docspell/restserver/conv/AddonValidationSupport.scala Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.conv
import cats.syntax.all._
import docspell.addons.AddonMeta
import docspell.backend.ops.AddonValidationError
import docspell.backend.ops.OAddons.AddonValidationResult
import docspell.common.Ident
import docspell.restserver.ws.{OutputEvent, OutputEventEncoder}
import docspell.store.records.RAddonArchive
trait AddonValidationSupport {
def validationErrorToMessage(e: AddonValidationError): String =
e match {
case AddonValidationError.AddonNotFound =>
"Addon not found."
case AddonValidationError.AddonExists(msg, _) =>
msg
case AddonValidationError.NotAnAddon(ex) =>
s"The url doesn't seem to be an addon: ${ex.getMessage}"
case AddonValidationError.InvalidAddon(msg) =>
s"The addon is not valid: $msg"
case AddonValidationError.AddonUnsupported(msg, _) =>
msg
case AddonValidationError.AddonsDisabled =>
"Addons are disabled in the config file."
case AddonValidationError.UrlUntrusted(_) =>
"This url doesn't belong to te set of trusted urls defined in the config file"
case AddonValidationError.DownloadFailed(ex) =>
s"Downloading the addon failed: ${ex.getMessage}"
case AddonValidationError.ImpureAddonsDisabled =>
s"Installing impure addons is disabled."
case AddonValidationError.RefreshLocalAddon =>
"Refreshing a local addon doesn't work."
}
def addonResultOutputEventEncoder(
collective: Ident
): OutputEventEncoder[AddonValidationResult[(RAddonArchive, AddonMeta)]] =
OutputEventEncoder.instance {
case Right((archive, _)) =>
OutputEvent.AddonInstalled(
collective,
"Addon installed",
None,
archive.id.some,
archive.originalUrl
)
case Left(error) =>
val msg = validationErrorToMessage(error)
OutputEvent.AddonInstalled(collective, msg, error.some, None, None)
}
}

4
modules/restserver/src/main/scala/docspell/restserver/http4s/Responses.scala
View File

@ -47,4 +47,8 @@ object Responses {
def notFoundRoute[F[_]: Sync]: HttpRoutes[F] =
HttpRoutes(_ => OptionT.pure(Response.notFound[F]))
def notFoundRoute[F[_]: Sync, A](body: A)(implicit
entityEncoder: EntityEncoder[F, A]
): HttpRoutes[F] =
HttpRoutes(_ => OptionT.pure(Response.notFound[F].withEntity(body)))
}

42
modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.http4s
import cats.effect._
import docspell.joexapi.model.BasicResult
import org.http4s.Response
import org.http4s.circe.CirceEntityEncoder._
import org.http4s.dsl.Http4sDsl
trait ThrowableResponseMapper {
implicit class EitherThrowableOps[A](self: Either[Throwable, A]) {
def rightAs[F[_]: Sync](f: A => F[Response[F]]): F[Response[F]] =
self.fold(ThrowableResponseMapper.toResponse[F], f)
def rightAs_[F[_]: Sync](r: => F[Response[F]]): F[Response[F]] =
self.fold(ThrowableResponseMapper.toResponse[F], _ => r)
}
}
object ThrowableResponseMapper {
def toResponse[F[_]: Sync](ex: Throwable): F[Response[F]] =
new Mapper[F].toResponse(ex)
private class Mapper[F[_]: Sync] extends Http4sDsl[F] {
def toResponse(ex: Throwable): F[Response[F]] =
ex match {
case _: IllegalArgumentException =>
BadRequest(BasicResult(false, ex.getMessage))
case _ =>
InternalServerError(BasicResult(false, ex.getMessage))
}
}
}

127
modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala Normal file
View File

@ -0,0 +1,127 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.routes
import cats.effect._
import cats.syntax.all._
import fs2.concurrent.Topic
import docspell.addons.AddonMeta
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.backend.ops.AddonValidationError
import docspell.common.Ident
import docspell.restapi.model._
import docspell.restserver.conv.AddonValidationSupport
import docspell.restserver.ws.{Background, OutputEvent}
import docspell.store.records.RAddonArchive
import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.Http4sDsl
import org.http4s.dsl.impl.FlagQueryParamMatcher
import org.http4s.{HttpRoutes, Response}
object AddonArchiveRoutes extends AddonValidationSupport {
def apply[F[_]: Async](
wsTopic: Topic[F, OutputEvent],
backend: BackendApp[F],
token: AuthToken
): HttpRoutes[F] = {
val dsl = new Http4sDsl[F] {}
import dsl._
implicit val wsOutputEnc = addonResultOutputEventEncoder(token.account.collective)
HttpRoutes.of {
case GET -> Root =>
for {
all <- backend.addons.getAllAddons(token.account.collective)
resp <- Ok(
AddonList(
all.map(r =>
Addon(r.id, r.name, r.version, r.description, r.originalUrl, r.created)
)
)
)
} yield resp
case req @ POST -> Root :? Sync(sync) =>
def create(r: Option[RAddonArchive]) =
IdResult(
true,
r.fold("Addon submitted for installation")(r =>
s"Addon installed: ${r.id.id}"
),
r.map(_.id).getOrElse(Ident.unsafe(""))
)
for {
input <- req.as[AddonRegister]
install = backend.addons.registerAddon(
token.account.collective,
input.url,
None
)
resp <-
if (sync)
install.flatMap(
_.fold(convertAddonValidationError[F], r => Ok(create(r._1.some)))
)
else Background(wsTopic)(install).flatMap(_ => Ok(create(None)))
} yield resp
case PUT -> Root / Ident(id) :? Sync(sync) =>
def create(r: Option[AddonMeta]) =
BasicResult(
true,
r.fold("Addon updated in background")(m =>
s"Addon updated: ${m.nameAndVersion}"
)
)
val update = backend.addons.refreshAddon(token.account.collective, id)
for {
resp <-
if (sync)
update.flatMap(
_.fold(
convertAddonValidationError[F],
r => Ok(create(r._2.some))
)
)
else Background(wsTopic)(update).flatMap(_ => Ok(create(None)))
} yield resp
case DELETE -> Root / Ident(id) =>
for {
flag <- backend.addons.deleteAddon(token.account.collective, id)
resp <-
if (flag) Ok(BasicResult(true, "Addon deleted"))
else NotFound(BasicResult(false, "Addon not found"))
} yield resp
}
}
def convertAddonValidationError[F[_]: Async](
e: AddonValidationError
): F[Response[F]] = {
val dsl = new Http4sDsl[F] {}
import dsl._
def failWith(msg: String): F[Response[F]] =
Ok(IdResult(false, msg, Ident.unsafe("")))
e match {
case AddonValidationError.AddonNotFound =>
NotFound(BasicResult(false, "Addon not found."))
case _ =>
failWith(validationErrorToMessage(e))
}
}
object Sync extends FlagQueryParamMatcher("sync")
}

39
modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.routes
import cats.effect.Async
import fs2.concurrent.Topic
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.restapi.model.BasicResult
import docspell.restserver.Config
import docspell.restserver.http4s.Responses
import docspell.restserver.ws.OutputEvent
import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityEncoder._
import org.http4s.server.Router
object AddonRoutes {
def apply[F[_]: Async](
cfg: Config,
wsTopic: Topic[F, OutputEvent],
backend: BackendApp[F],
token: AuthToken
): HttpRoutes[F] =
if (cfg.backend.addons.enabled)
Router(
"archive" -> AddonArchiveRoutes(wsTopic, backend, token),
"run-config" -> AddonRunConfigRoutes(backend, token),
"run" -> AddonRunRoutes(backend, token)
)
else
Responses.notFoundRoute(BasicResult(false, "Addons disabled"))
}

110
modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala Normal file
View File

@ -0,0 +1,110 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.routes
import cats.data.NonEmptyList
import cats.effect._
import cats.syntax.all._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.backend.ops.OAddons
import docspell.common.Ident
import docspell.restapi.model._
import docspell.restserver.http4s.ThrowableResponseMapper
import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.Http4sDsl
object AddonRunConfigRoutes {
def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F] with ThrowableResponseMapper {}
import dsl._
HttpRoutes.of {
case GET -> Root =>
for {
all <- backend.addons.getAllAddonRunConfigs(token.account.collective)
resp <- Ok(AddonRunConfigList(all.map(convertInfoTask)))
} yield resp
case req @ POST -> Root =>
for {
input <- req.as[AddonRunConfig]
data = convertInsertTask(Ident.unsafe(""), input)
res <- data.flatTraverse(in =>
backend.addons
.upsertAddonRunConfig(token.account.collective, in)
.map(_.leftMap(_.message))
)
resp <- res.fold(
msg => Ok(BasicResult(false, msg)),
id => Ok(IdResult(true, s"Addon run config added", id))
)
} yield resp
case req @ PUT -> Root / Ident(id) =>
for {
input <- req.as[AddonRunConfig]
data = convertInsertTask(id, input)
res <- data.flatTraverse(in =>
backend.addons
.upsertAddonRunConfig(token.account.collective, in)
.map(_.leftMap(_.message))
)
resp <- res.fold(
msg => Ok(BasicResult(false, msg)),
id => Ok(IdResult(true, s"Addon run config updated", id))
)
} yield resp
case DELETE -> Root / Ident(id) =>
for {
flag <- backend.addons.deleteAddonRunConfig(token.account.collective, id)
resp <-
if (flag) Ok(BasicResult(true, "Addon task deleted"))
else NotFound(BasicResult(false, "Addon task not found"))
} yield resp
}
}
def convertInsertTask(
id: Ident,
t: AddonRunConfig
): Either[String, OAddons.AddonRunInsert] =
for {
tr <- NonEmptyList
.fromList(t.trigger)
.toRight("At least one trigger is required")
ta <- NonEmptyList
.fromList(t.addons)
.toRight("At least one addon is required")
res = OAddons.AddonRunInsert(
id,
t.name,
t.enabled,
t.userId,
t.schedule,
tr,
ta.map(e => OAddons.AddonArgs(e.addonId, e.args))
)
} yield res
def convertInfoTask(t: OAddons.AddonRunInfo): AddonRunConfig =
AddonRunConfig(
id = t.id,
name = t.name,
enabled = t.enabled,
userId = t.userId,
schedule = t.schedule,
trigger = t.triggered,
addons = t.addons.map { case (ra, raa) =>
AddonRef(raa.addonId, ra.name, ra.version, ra.description, raa.args)
}
)
}

40
modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.routes
import cats.data.NonEmptyList
import cats.effect._
import cats.syntax.all._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.restapi.model._
import docspell.restserver.http4s.ThrowableResponseMapper
import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityCodec._
import org.http4s.dsl.Http4sDsl
object AddonRunRoutes {
def apply[F[_]: Async](backend: BackendApp[F], token: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F] with ThrowableResponseMapper {}
import dsl._
HttpRoutes.of { case req @ POST -> Root / "existingitem" =>
for {
input <- req.as[AddonRunExistingItem]
_ <- backend.addons.runAddonForItem(
token.account,
NonEmptyList(input.itemId, input.additionalItems),
input.addonRunConfigIds.toSet
)
resp <- Ok(BasicResult(true, "Job for running addons submitted."))
} yield resp
}
}
}

6
modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala
View File

@ -117,7 +117,11 @@ object ItemMultiRoutes extends NonEmptyListSupport with MultiIdSupport {
for {
json <- req.as[ItemsAndRef]
items <- requireNonEmpty(json.items)
res <- backend.item.setFolderMultiple(items, json.ref, user.account.collective)
res <- backend.item.setFolderMultiple(
items,
json.ref.map(_.id),
user.account.collective
)
resp <- Ok(Conversions.basicResult(res, "Folder updated"))
} yield resp

2
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@ -218,7 +218,7 @@ object ItemRoutes {
case req @ PUT -> Root / Ident(id) / "folder" =>
for {
idref <- req.as[OptionalId]
res <- backend.item.setFolder(id, idref.id, user.account.collective)
res <- backend.item.setFolder(id, idref.id.map(_.id), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Folder updated"))
} yield resp

6
modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala
View File

@ -29,7 +29,8 @@ case class Flags(
downloadAllMaxFiles: Int,
downloadAllMaxSize: ByteSize,
uiVersion: Int,
openIdAuth: List[Flags.OpenIdAuth]
openIdAuth: List[Flags.OpenIdAuth],
addonsEnabled: Boolean
)
object Flags {
@ -47,7 +48,8 @@ object Flags {
cfg.downloadAll.maxFiles,
cfg.downloadAll.maxSize,
uiVersion,
cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display))
cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display)),
cfg.backend.addons.enabled
)
final case class OpenIdAuth(provider: Ident, name: String)

43
modules/restserver/src/main/scala/docspell/restserver/ws/Background.scala Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.ws
import cats.effect._
import cats.syntax.all._
import fs2.concurrent.Topic
import docspell.logging.Logger
/** Asynchronous operations that run on the rest-server can communicate their results via
* websocket.
*/
object Background {
// TODO avoid resubmitting same stuff
def apply[F[_]: Async, A](
wsTopic: Topic[F, OutputEvent],
logger: Option[Logger[F]] = None
)(run: F[A])(implicit enc: OutputEventEncoder[A]): F[Unit] = {
val log = logger.getOrElse(docspell.logging.getLogger[F])
Async[F]
.background(run)
.use(
_.flatMap(
_.fold(
log.warn("The background operation has been cancelled!"),
ex => log.error(ex)("Error running background operation!"),
event =>
event
.map(enc.encode)
.flatTap(ev => log.info(s"Sending response from async operation: $ev"))
.flatMap(wsTopic.publish1)
.void
)
)
)
}
}

24
modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala
View File

@ -7,6 +7,7 @@
package docspell.restserver.ws
import docspell.backend.auth.AuthToken
import docspell.backend.ops.AddonValidationError
import docspell.common._
import io.circe._
@ -55,6 +56,29 @@ object OutputEvent {
Msg("jobs-waiting", count).asJson
}
final case class AddonInstalled(
collective: Ident,
message: String,
error: Option[AddonValidationError],
addonId: Option[Ident],
originalUrl: Option[LenientUri]
) extends OutputEvent {
def forCollective(token: AuthToken) =
token.account.collective == collective
override def asJson =
Msg(
"addon-installed",
Map(
"success" -> error.isEmpty.asJson,
"error" -> error.asJson,
"addonId" -> addonId.asJson,
"addonUrl" -> originalUrl.asJson,
"message" -> message.asJson
)
).asJson
}
private case class Msg[A](tag: String, content: A)
private object Msg {
implicit def jsonEncoder[A: Encoder]: Encoder[Msg[A]] =

18
modules/restserver/src/main/scala/docspell/restserver/ws/OutputEventEncoder.scala Normal file
View File

@ -0,0 +1,18 @@
/*
* Copyright 2020 Eike K. & Contributors
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
package docspell.restserver.ws
trait OutputEventEncoder[A] {
def encode(a: A): OutputEvent
}
object OutputEventEncoder {
def apply[A](implicit e: OutputEventEncoder[A]): OutputEventEncoder[A] = e
def instance[A](f: A => OutputEvent): OutputEventEncoder[A] =
(a: A) => f(a)
}