TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve performance of zip/unzip

Browse Source 
Adds tests and includes some cleanup
This commit is contained in:
eikek
2022-06-18 16:37:38 +02:00
parent 483dbf5d2b
commit 6cef9d4f07
24 changed files with 711 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modules/joex/src/main/scala/docspell/joex/addon/GenericItemAddonTask.scala
View File

@ -15,14 +15,14 @@ import docspell.addons.{AddonTriggerType, InputEnv, Middleware}
import docspell.backend.joex.AddonOps.ExecResult
import docspell.backend.joex.{AddonOps, LoggerExtension}
import docspell.common._
import docspell.files.FileSupport
import docspell.common.syntax.file._
import docspell.joex.process.ItemData
import docspell.logging.Logger
import docspell.scheduler.Task
import docspell.store.Store
import docspell.store.queries.QAttachment
object GenericItemAddonTask extends LoggerExtension with FileSupport {
object GenericItemAddonTask extends LoggerExtension {
private val itemSubdir = "item"
private val itemDataJson = s"$itemSubdir/item-data.json"

4
modules/joex/src/main/scala/docspell/joex/download/DownloadZipTask.scala
View File

@ -16,7 +16,7 @@ import docspell.backend.ops.ODownloadAll
import docspell.backend.ops.ODownloadAll.model.DownloadSummary
import docspell.backend.task.DownloadZipArgs
import docspell.common._
import docspell.files.Zip
import docspell.common.util.Zip
import docspell.scheduler.Task
import docspell.store.Store
import docspell.store.queries.{ItemFileMeta, QItem}
@ -50,7 +50,7 @@ object DownloadZipTask {
val storeZipFile =
allFiles
.through(Zip.zip(ctx.logger, chunkSize))
.through(Zip[F](ctx.logger.some).zip(chunkSize))
.through(
store.fileRepo.save(
ctx.args.accountId.collective,

7
modules/joex/src/main/scala/docspell/joex/multiupload/MultiUploadArchiveTask.scala
View File

@ -14,7 +14,7 @@ import fs2.Stream
import docspell.backend.JobFactory
import docspell.common._
import docspell.files.Zip
import docspell.common.util.Zip
import docspell.logging.Logger
import docspell.scheduler._
import docspell.store.Store
@ -44,7 +44,7 @@ object MultiUploadArchiveTask {
extractZip(store, ctx.args)(file)
.evalTap(entry =>
ctx.logger.debug(
s"Create job for entry: ${entry.files.flatMap(_.name)}"
s"Create job for entry: ${entry.files.flatMap(_.name).mkString(", ")}"
)
)
.evalMap(makeJob[F](ctx, jobStore))
@ -109,7 +109,8 @@ object MultiUploadArchiveTask {
)(file: ProcessItemArgs.File): Stream[F, ProcessItemArgs] =
store.fileRepo
.getBytes(file.fileMetaId)
.through(Zip.unzip[F](8192, args.meta.fileFilter.getOrElse(Glob.all)))
.through(Zip[F]().unzip(glob = args.meta.fileFilter.getOrElse(Glob.all)))
.through(Binary.toBinary[F])
.flatMap { entry =>
val hint = MimeTypeHint(entry.name.some, entry.mime.asString.some)
entry.data

5
modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
View File

@ -16,7 +16,7 @@ import cats.kernel.Order
import fs2.Stream
import docspell.common._
import docspell.files.Zip
import docspell.common.util.Zip
import docspell.joex.mail._
import docspell.scheduler._
import docspell.store.Store
@ -146,7 +146,8 @@ object ExtractArchive {
val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all)
ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *>
zipData
.through(Zip.unzip[F](8192, glob))
.through(Zip[F](ctx.logger.some).unzip(glob = glob))
.through(Binary.toBinary[F])
.zipWithIndex
.flatMap(handleEntry(ctx, store, ra, pos, archive, None))
.foldMonoid