Download multiple files as zip

modules/restserver/src/main/resources/reference.conf

@@ -90,6 +90,15 @@ docspell.server {
     }
   }
 
+  # Settings for "download as zip"
+  download-all {
+    # How many files to allow in a zip.
+    max-files = 500
+
+    # The maximum (uncompressed) size of the zip file contents.
+    max-size = 1400M
+  }
+
   # Configures OpenID Connect (OIDC) or OAuth2 authentication. Only
   # the "Authorization Code Flow" is supported.
   #

modules/restserver/src/main/scala/docspell/restserver/Config.scala

@@ -14,7 +14,7 @@ import docspell.ftssolr.SolrConfig
 import docspell.logging.LogConfig
 import docspell.oidc.ProviderConfig
 import docspell.pubsub.naive.PubSubConfig
-import docspell.restserver.Config.{OpenIdConfig, ServerOptions}
+import docspell.restserver.Config.{DownloadAllCfg, OpenIdConfig, ServerOptions}
 import docspell.restserver.auth.OpenId
 import docspell.restserver.http4s.InternalHeader
 
@@ -36,7 +36,8 @@ case class Config(
     maxNoteLength: Int,
     fullTextSearch: Config.FullTextSearch,
     adminEndpoint: Config.AdminEndpoint,
-    openid: List[OpenIdConfig]
+    openid: List[OpenIdConfig],
+    downloadAll: DownloadAllCfg
 ) {
   def openIdEnabled: Boolean =
     openid.exists(_.enabled)
@@ -51,6 +52,7 @@ case class Config(
 }
 
 object Config {
+  case class DownloadAllCfg(maxFiles: Int, maxSize: ByteSize)
 
   case class ServerOptions(
       responseTimeout: Duration,

modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala

@@ -93,8 +93,10 @@ final class RestAppImpl[F[_]: Async](
       "search" -> ShareSearchRoutes(backend, config, token),
       "attachment" -> ShareAttachmentRoutes(backend, token),
       "item" -> ShareItemRoutes(backend, token),
-      "clientSettings" -> ClientSettingsRoutes.share(backend, token)
+      "clientSettings" -> ClientSettingsRoutes.share(backend, token),
+      "downloadAll" -> DownloadAllRoutes.forShare(config.downloadAll, backend, token)
     )
+
   def openRoutes(
       client: Client[F]
   ): HttpRoutes[F] =
@@ -149,7 +151,8 @@ final class RestAppImpl[F[_]: Async](
       "customfield" -> CustomFieldRoutes(backend, token),
       "clientSettings" -> ClientSettingsRoutes(backend, token),
       "notification" -> NotificationRoutes(config, backend, token),
-      "querybookmark" -> BookmarkRoutes(backend, token)
+      "querybookmark" -> BookmarkRoutes(backend, token),
+      "downloadAll" -> DownloadAllRoutes(config.downloadAll, backend, token)
     )
 
 }

modules/restserver/src/main/scala/docspell/restserver/http4s/BinaryUtil.scala

@@ -11,7 +11,7 @@ import cats.data.OptionT
 import cats.effect._
 import cats.implicits._
 
-import docspell.backend.ops.OItemSearch.{AttachmentData, AttachmentPreviewData}
+import docspell.backend.ops.OItemSearch.{AttachmentPreviewData, BinaryData}
 import docspell.backend.ops._
 import docspell.restapi.model.BasicResult
 import docspell.restserver.http4s.{QueryParam => QP}
@@ -27,7 +27,7 @@ import org.typelevel.ci.CIString
 object BinaryUtil {
 
   def respond[F[_]: Async](dsl: Http4sDsl[F], req: Request[F])(
-      fileData: Option[AttachmentData[F]]
+      fileData: Option[BinaryData[F]]
   ): F[Response[F]] = {
     import dsl._
 
@@ -42,7 +42,7 @@ object BinaryUtil {
   }
 
   def respondHead[F[_]: Async](dsl: Http4sDsl[F])(
-      fileData: Option[AttachmentData[F]]
+      fileData: Option[BinaryData[F]]
   ): F[Response[F]] = {
     import dsl._
 

modules/restserver/src/main/scala/docspell/restserver/routes/DownloadAllRoutes.scala

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver.routes
+
+import cats.data.{Kleisli, OptionT}
+import cats.effect._
+import cats.syntax.all._
+
+import docspell.backend.BackendApp
+import docspell.backend.auth.{AuthToken, ShareToken}
+import docspell.backend.ops.ODownloadAll.model._
+import docspell.backend.ops.OShare.ShareQuery
+import docspell.common.{DownloadAllType, Ident}
+import docspell.joexapi.model.BasicResult
+import docspell.query.ItemQuery
+import docspell.restapi.model.{DownloadAllRequest, DownloadAllSummary}
+import docspell.restserver.Config.DownloadAllCfg
+import docspell.restserver.conv.Conversions
+import docspell.restserver.http4s.BinaryUtil
+
+import org.http4s.circe.CirceEntityCodec._
+import org.http4s.dsl.Http4sDsl
+import org.http4s.{HttpRoutes, Request}
+
+object DownloadAllRoutes {
+
+  def forShare[F[_]: Async](
+      cfg: DownloadAllCfg,
+      backend: BackendApp[F],
+      token: ShareToken
+  ): HttpRoutes[F] = {
+    val dsl = new Http4sDsl[F] {}
+    import dsl._
+
+    val find: Kleisli[OptionT[F, *], Request[F], ShareQuery] =
+      Kleisli(_ => backend.share.findShareQuery(token.id))
+
+    find.flatMap { share =>
+      HttpRoutes.of[F] {
+        case req @ POST -> Root / "prefetch" =>
+          for {
+            input <- req.as[DownloadAllRequest]
+            query = ItemQuery.Expr.and(share.query.expr, input.query.expr)
+            result <- backend.downloadAll.getSummary(
+              share.account,
+              DownloadRequest(
+                ItemQuery(query, None),
+                DownloadAllType.Converted,
+                cfg.maxFiles,
+                cfg.maxSize
+              )
+            )
+            resp <- Ok(convertSummary(result))
+          } yield resp
+
+        case req @ POST -> Root / "submit" =>
+          for {
+            input <- req.as[DownloadAllRequest]
+            query = ItemQuery.Expr.and(share.query.expr, input.query.expr)
+            result <- backend.downloadAll.submit(
+              share.account,
+              DownloadRequest(
+                ItemQuery(query, None),
+                DownloadAllType.Converted,
+                cfg.maxFiles,
+                cfg.maxSize
+              )
+            )
+            resp <- Ok(convertSummary(result))
+          } yield resp
+
+        case req @ GET -> Root / "file" / Ident(id) =>
+          for {
+            data <- backend.downloadAll.getFile(share.account.collective, id)
+            resp <- BinaryUtil.respond(dsl, req)(data)
+          } yield resp
+      }
+    }
+  }
+
+  def apply[F[_]: Async](
+      cfg: DownloadAllCfg,
+      backend: BackendApp[F],
+      token: AuthToken
+  ): HttpRoutes[F] = {
+    val dsl = new Http4sDsl[F] {}
+    import dsl._
+
+    HttpRoutes.of {
+      case req @ POST -> Root / "prefetch" =>
+        for {
+          input <- req.as[DownloadAllRequest]
+          result <- backend.downloadAll.getSummary(
+            token.account,
+            DownloadRequest(input.query, input.fileType, cfg.maxFiles, cfg.maxSize)
+          )
+          resp <- Ok(convertSummary(result))
+        } yield resp
+
+      case req @ POST -> Root / "submit" =>
+        for {
+          input <- req.as[DownloadAllRequest]
+          result <- backend.downloadAll.submit(
+            token.account,
+            DownloadRequest(input.query, input.fileType, cfg.maxFiles, cfg.maxSize)
+          )
+          resp <- Ok(convertSummary(result))
+        } yield resp
+
+      case req @ GET -> Root / "file" / Ident(id) =>
+        for {
+          data <- backend.downloadAll.getFile(token.account.collective, id)
+          resp <- BinaryUtil.respond(dsl, req)(data)
+        } yield resp
+
+      case HEAD -> Root / "file" / Ident(id) =>
+        for {
+          data <- backend.downloadAll.getFile(token.account.collective, id)
+          resp <- BinaryUtil.respondHead(dsl)(data)
+        } yield resp
+
+      case DELETE -> Root / "file" / Ident(id) =>
+        for {
+          _ <- backend.downloadAll.deleteFile(id)
+          resp <- Ok(BasicResult(true, "File deleted."))
+        } yield resp
+
+      case PUT -> Root / "cancel" / Ident(id) =>
+        for {
+          res <- backend.downloadAll.cancelDownload(token.account, id)
+          resp <- Ok(Conversions.basicResult(res))
+        } yield resp
+    }
+  }
+
+  private def convertSummary(result: DownloadSummary): DownloadAllSummary =
+    DownloadAllSummary(
+      id = result.id,
+      fileCount = result.fileCount,
+      uncompressedSize = result.uncompressedSize,
+      state = result.state
+    )
+}

modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala

@@ -7,7 +7,7 @@
 package docspell.restserver.webapp
 
 import docspell.backend.signup.{Config => SignupConfig}
-import docspell.common.{Ident, LenientUri}
+import docspell.common.{ByteSize, Ident, LenientUri}
 import docspell.restserver.{BuildInfo, Config}
 
 import io.circe._
@@ -26,6 +26,8 @@ case class Flags(
     maxPageSize: Int,
     maxNoteLength: Int,
     showClassificationSettings: Boolean,
+    downloadAllMaxFiles: Int,
+    downloadAllMaxSize: ByteSize,
     uiVersion: Int,
     openIdAuth: List[Flags.OpenIdAuth]
 )
@@ -42,6 +44,8 @@ object Flags {
       cfg.maxItemPageSize,
       cfg.maxNoteLength,
       cfg.showClassificationSettings,
+      cfg.downloadAll.maxFiles,
+      cfg.downloadAll.maxSize,
       uiVersion,
       cfg.openid.filter(_.enabled).map(c => OpenIdAuth(c.provider.providerId, c.display))
     )
@@ -63,6 +67,9 @@ object Flags {
   implicit val jsonEncoder: Encoder[Flags] =
     deriveEncoder[Flags]
 
+  implicit def yamuscaByteSizeConverter: ValueConverter[ByteSize] =
+    ValueConverter.of(sz => Value.fromString(sz.bytes.toString))
+
   implicit def yamuscaIdentConverter: ValueConverter[Ident] =
     ValueConverter.of(id => Value.fromString(id.id))
   implicit def yamuscaOpenIdAuthConverter: ValueConverter[OpenIdAuth] =