Download multiple files as zip

modules/store/src/main/resources/db/migration/h2/V1.35.0__download_query.sql

@@ -0,0 +1,11 @@
+CREATE TABLE "download_query"(
+  "id" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "file_id" varchar(254) not null,
+  "file_count" int not null,
+  "created" timestamp not null,
+  "last_access" timestamp,
+  "access_count" int not null,
+  foreign key ("cid") references "collective"("cid"),
+  foreign key ("file_id") references "filemeta"("file_id")
+);

modules/store/src/main/resources/db/migration/mariadb/V1.35.0__download_query.sql

@@ -0,0 +1,11 @@
+CREATE TABLE `download_query`(
+  `id` varchar(254) not null primary key,
+  `cid` varchar(254) not null,
+  `file_id` varchar(254) not null,
+  `file_count` int not null,
+  `created` timestamp not null,
+  `last_access` timestamp,
+  `access_count` int not null,
+  foreign key (`cid`) references `collective`(`cid`),
+  foreign key (`file_id`) references `filemeta`(`file_id`)
+);

modules/store/src/main/resources/db/migration/postgresql/V1.35.0__download_query.sql

@@ -0,0 +1,11 @@
+CREATE TABLE "download_query"(
+  "id" varchar(254) not null primary key,
+  "cid" varchar(254) not null,
+  "file_id" varchar(254) not null,
+  "file_count" int not null,
+  "created" timestamp not null,
+  "last_access" timestamp,
+  "access_count" int not null,
+  foreign key ("cid") references "collective"("cid"),
+  foreign key ("file_id") references "filemeta"("file_id")
+);

modules/store/src/main/scala/docspell/store/qb/DSL.scala

@@ -47,6 +47,12 @@ trait DSL extends DoobieMeta {
   def select(e: SelectExpr, es: SelectExpr*): Nel[SelectExpr] =
     Nel(e, es.toList)
 
+  def combineNel[A](e: Nel[A], more: Nel[A]*): Nel[A] =
+    Nel
+      .fromFoldable(more)
+      .map(tail => tail.prepend(e).flatMap(identity))
+      .getOrElse(e)
+
   def select(c: Column[_], cs: Column[_]*): Nel[SelectExpr] =
     Nel(c, cs.toList).map(col => SelectExpr.SelectColumn(col, None))
 

modules/store/src/main/scala/docspell/store/queries/ItemFileMeta.scala

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.queries
+
+import docspell.common._
+import docspell.store.records.RFileMeta
+
+/** Almost like [[ListItem]] but without notes and at file level. */
+final case class ItemFileMeta(
+    id: Ident,
+    name: String,
+    state: ItemState,
+    date: Timestamp,
+    dueDate: Option[Timestamp],
+    source: String,
+    direction: Direction,
+    created: Timestamp,
+    corrOrg: Option[IdRef],
+    corrPerson: Option[IdRef],
+    concPerson: Option[IdRef],
+    concEquip: Option[IdRef],
+    folder: Option[IdRef],
+    fileName: Option[String],
+    fileMeta: RFileMeta
+)

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -38,9 +38,11 @@ object QItem {
   private val cf = RCustomField.as("cf")
   private val cv = RCustomFieldValue.as("cvf")
   private val a = RAttachment.as("a")
+  private val as = RAttachmentSource.as("ras")
   private val m = RAttachmentMeta.as("m")
   private val tag = RTag.as("t")
   private val ti = RTagItem.as("ti")
+  private val meta = RFileMeta.as("fmeta")
 
   def countAttachmentsAndItems(items: Nel[Ident]): ConnectionIO[Int] =
     Select(count(a.id).s, from(a), a.itemId.in(items)).build
@@ -176,6 +178,87 @@ object QItem {
     )
   }
 
+  private def findFilesQuery(
+      q: Query,
+      ftype: DownloadAllType,
+      today: LocalDate,
+      maxFiles: Int
+  ): Select =
+    findItemsBase(q.fix, today, 0)
+      .changeFrom(_.innerJoin(a, a.itemId === i.id).innerJoin(as, a.id === as.id))
+      .changeFrom(from =>
+        ftype match {
+          case DownloadAllType.Converted =>
+            from.innerJoin(meta, meta.id === a.fileId)
+          case DownloadAllType.Original =>
+            from.innerJoin(meta, meta.id === as.fileId)
+        }
+      )
+      .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
+      .limit(maxFiles)
+
+  def findFiles(
+      q: Query,
+      ftype: DownloadAllType,
+      today: LocalDate,
+      maxFiles: Int,
+      chunkSize: Int
+  ): Stream[ConnectionIO, RFileMeta] = {
+    val query = findFilesQuery(q, ftype, today, maxFiles)
+      .withSelect(
+        meta.all.map(_.s).append(coalesce(i.itemDate.s, i.created.s).s)
+      )
+
+    query.build
+      .query[RFileMeta]
+      .streamWithChunkSize(chunkSize)
+  }
+
+  def findFilesDetailed(
+      q: Query,
+      ftype: DownloadAllType,
+      today: LocalDate,
+      maxFiles: Int,
+      chunkSize: Int
+  ): Stream[ConnectionIO, ItemFileMeta] = {
+    val fname = ftype match {
+      case DownloadAllType.Converted => a.name
+      case DownloadAllType.Original  => as.name
+    }
+
+    val query = findFilesQuery(q, ftype, today, maxFiles)
+      .withSelect(
+        combineNel(
+          select(
+            i.id.s,
+            i.name.s,
+            i.state.s,
+            coalesce(i.itemDate.s, i.created.s).s,
+            i.dueDate.s,
+            i.source.s,
+            i.incoming.s,
+            i.created.s,
+            org.oid.s,
+            org.name.s,
+            pers0.pid.s,
+            pers0.name.s,
+            pers1.pid.s,
+            pers1.name.s,
+            equip.eid.s,
+            equip.name.s,
+            f.id.s,
+            f.name.s
+          ),
+          select(fname.s),
+          select(meta.all)
+        )
+      )
+
+    query.build
+      .query[ItemFileMeta]
+      .streamWithChunkSize(chunkSize)
+  }
+
   def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery.Expr): Condition = {
     val tables = Tables(i, org, pers0, pers1, equip, f, a, m, AttachCountTable("cta"))
     ItemQueryGenerator.fromExpr(today, tables, coll)(q)

modules/store/src/main/scala/docspell/store/records/RDownloadQuery.scala

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.records
+
+import cats.data.NonEmptyList
+
+import docspell.common._
+import docspell.store.qb.DSL._
+import docspell.store.qb._
+
+import doobie._
+import doobie.implicits._
+
+final case class RDownloadQuery(
+    id: Ident,
+    cid: Ident,
+    fileId: FileKey,
+    fileCount: Int,
+    created: Timestamp,
+    lastAccess: Option[Timestamp],
+    accessCount: Int
+) {}
+
+object RDownloadQuery {
+
+  case class Table(alias: Option[String]) extends TableDef {
+    val tableName = "download_query"
+
+    val id: Column[Ident] = Column("id", this)
+    val cid: Column[Ident] = Column("cid", this)
+    val fileId: Column[FileKey] = Column("file_id", this)
+    val fileCount: Column[Int] = Column("file_count", this)
+    val created: Column[Timestamp] = Column("created", this)
+    val lastAccess: Column[Timestamp] = Column("last_access", this)
+    val accessCount: Column[Int] = Column("access_count", this)
+
+    val all: NonEmptyList[Column[_]] =
+      NonEmptyList.of(id, cid, fileId, fileCount, created, lastAccess, accessCount)
+  }
+
+  def as(alias: String): Table =
+    Table(Some(alias))
+
+  val T = Table(None)
+
+  def insert(r: RDownloadQuery): ConnectionIO[Int] =
+    DML.insert(
+      T,
+      T.all,
+      sql"${r.id},${r.cid},${r.fileId},${r.fileCount},${r.created},${r.lastAccess},${r.accessCount}"
+    )
+
+  def existsById(id: Ident): ConnectionIO[Boolean] =
+    Select(select(count(T.id)), from(T), T.id === id).build.query[Int].unique.map(_ > 0)
+
+  def findById(id: Ident): ConnectionIO[Option[(RDownloadQuery, RFileMeta)]] = {
+    val dq = RDownloadQuery.as("dq")
+    val fm = RFileMeta.as("fm")
+    Select(
+      select(dq.all, fm.all),
+      from(dq).innerJoin(fm, fm.id === dq.fileId),
+      dq.id === id
+    ).build
+      .query[(RDownloadQuery, RFileMeta)]
+      .option
+  }
+
+  def updateAccess(id: Ident, ts: Timestamp): ConnectionIO[Int] =
+    DML.update(
+      T,
+      T.id === id,
+      DML.set(
+        T.lastAccess.setTo(ts),
+        T.accessCount.increment(1)
+      )
+    )
+
+  def updateAccessNow(id: Ident): ConnectionIO[Int] =
+    Timestamp
+      .current[ConnectionIO]
+      .flatMap(updateAccess(id, _))
+
+  def deleteById(id: Ident): ConnectionIO[Int] =
+    DML.delete(T, T.id === id)
+
+  def deleteByFileKey(fkey: FileKey): ConnectionIO[Int] =
+    DML.delete(T, T.fileId === fkey)
+
+  def findOlderThan(ts: Timestamp, batch: Int): ConnectionIO[List[FileKey]] =
+    Select(
+      select(T.fileId),
+      from(T),
+      T.lastAccess.isNull || T.lastAccess < ts
+    ).limit(batch)
+      .build
+      .query[FileKey]
+      .to[List]
+}

modules/store/src/main/scala/docspell/store/records/RFileMeta.scala

@@ -11,7 +11,7 @@ import cats.implicits._
 import fs2.Stream
 
 import docspell.common.{FileKey, _}
-import docspell.store.file.BinnyUtils
+import docspell.store.file.{BinnyUtils, FileMetadata}
 import docspell.store.qb.DSL._
 import docspell.store.qb._
 
@@ -25,7 +25,10 @@ final case class RFileMeta(
     mimetype: MimeType,
     length: ByteSize,
     checksum: ByteVector
-)
+) {
+  def toFileMetadata: FileMetadata =
+    FileMetadata(id, created, mimetype, length, checksum)
+}
 
 object RFileMeta {
   final case class Table(alias: Option[String]) extends TableDef {