Improve performance of zip/unzip

Adds tests and includes some cleanup

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -12,7 +12,8 @@ import fs2.Stream
 import fs2.io.file.{Files, Path}
 
 import docspell.common._
-import docspell.files.Zip
+import docspell.common.syntax.file._
+import docspell.common.util.{Directory, Zip}
 
 final case class AddonArchive(url: LenientUri, name: String, version: String) {
   def nameAndVersion: String =
@@ -36,8 +37,8 @@ final case class AddonArchive(url: LenientUri, name: String, version: String) {
         case false =>
           Files[F].createDirectories(target) *>
             reader(url)
-              .through(Zip.unzip(8192, glob))
-              .through(Zip.saveTo(logger, target, moveUp = true))
+              .through(Zip[F](logger.some).unzip(glob = glob, targetDir = target.some))
+              .evalTap(_ => Directory.unwrapSingle[F](logger, target))
               .compile
               .drain
               .as(target)
@@ -72,12 +73,13 @@ object AddonArchive {
       archive: Either[Path, Stream[F, Byte]]
   ): F[(Boolean, Boolean)] = {
     val files = Files[F]
+    val logger = docspell.logging.getLogger[F]
     def forPath(path: Path): F[(Boolean, Boolean)] =
       (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled
 
     def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] =
       data
-        .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix")))
+        .through(Zip[F](logger.some).unzip(glob = Glob("Dockerfile|flake.nix")))
         .collect {
           case bin if bin.name == "Dockerfile" => (true, false)
           case bin if bin.name == "flake.nix"  => (false, true)

modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala

@@ -14,6 +14,7 @@ import fs2.io.file._
 
 import docspell.common.UrlReader
 import docspell.common.exec.Env
+import docspell.common.util.Directory
 import docspell.logging.Logger
 
 trait AddonExecutor[F[_]] {

modules/addonlib/src/main/scala/docspell/addons/AddonMeta.scala

@@ -15,7 +15,8 @@ import fs2.Stream
 import fs2.io.file.{Files, Path}
 
 import docspell.common.Glob
-import docspell.files.Zip
+import docspell.common.syntax.file._
+import docspell.common.util.Zip
 
 import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
 import io.circe.yaml.{parser => YamlParser}
@@ -153,6 +154,12 @@ object AddonMeta {
       .map(fromJsonString)
       .rethrow
 
+  def fromJsonFile[F[_]: Sync](file: Path): F[AddonMeta] =
+    Sync[F]
+      .blocking(java.nio.file.Files.readString(file.toNioPath))
+      .map(fromJsonString)
+      .rethrow
+
   def fromYamlString(str: String): Either[Throwable, AddonMeta] =
     YamlParser.parse(str).flatMap(_.as[AddonMeta])
 
@@ -164,6 +171,13 @@ object AddonMeta {
       .map(fromYamlString)
       .rethrow
 
+  def fromYamlFile[F[_]: Sync](file: Path): F[AddonMeta] =
+    Sync[F]
+      .blocking(YamlParser.parse(java.nio.file.Files.newBufferedReader(file.toNioPath)))
+      .rethrow
+      .map(_.as[AddonMeta])
+      .rethrow
+
   def findInDirectory[F[_]: Sync: Files](dir: Path): F[AddonMeta] = {
     val logger = docspell.logging.getLogger[F]
     val jsonFile = dir / "docspell-addon.json"
@@ -194,18 +208,22 @@ object AddonMeta {
   }
 
   def findInZip[F[_]: Async](zipFile: Stream[F, Byte]): F[AddonMeta] = {
+    val logger = docspell.logging.getLogger[F]
     val fail: F[AddonMeta] = Async[F].raiseError(
       new FileNotFoundException(
         s"No docspell-addon.{yaml|json} file found in zip!"
       )
     )
     zipFile
-      .through(Zip.unzip(8192, Glob("docspell-addon.*|**/docspell-addon.*")))
-      .filter(bin => !bin.name.endsWith("/"))
+      .through(
+        Zip[F](logger.some).unzip(glob = Glob("docspell-addon.*|**/docspell-addon.*"))
+      )
+      .filter(file => !file.name.endsWith("/"))
       .flatMap { bin =>
-        if (bin.extensionIn(Set("json"))) Stream.eval(AddonMeta.fromJsonBytes(bin.data))
-        else if (bin.extensionIn(Set("yaml", "yml")))
-          Stream.eval(AddonMeta.fromYamlBytes(bin.data))
+        val ext = bin.extension
+        if (ext.equalsIgnoreCase("json")) Stream.eval(AddonMeta.fromJsonFile(bin))
+        else if (Set("yaml", "yml").contains(ext.toLowerCase))
+          Stream.eval(AddonMeta.fromYamlFile(bin))
         else Stream.empty
       }
       .take(1)

modules/addonlib/src/main/scala/docspell/addons/Directory.scala

@@ -1,74 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.addons
-
-import cats.effect._
-import cats.syntax.all._
-import cats.{Applicative, Monad}
-import fs2.io.file.{Files, Path, PosixPermissions}
-
-object Directory {
-
-  def create[F[_]: Files: Applicative](dir: Path): F[Path] =
-    Files[F]
-      .createDirectories(dir, PosixPermissions.fromOctal("777"))
-      .as(dir)
-
-  def createAll[F[_]: Files: Applicative](dir: Path, dirs: Path*): F[Unit] =
-    (dir :: dirs.toList).traverse_(Files[F].createDirectories(_))
-
-  def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
-    List(
-      Files[F].isDirectory(dir),
-      Files[F].list(dir).take(1).compile.last.map(_.isDefined)
-    ).sequence.map(_.forall(identity))
-
-  def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
-    nonEmpty(dir).map(b => !b)
-
-  def temp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
-    for {
-      _ <- Resource.eval(Files[F].createDirectories(parent))
-      d <- mkTemp(parent, prefix)
-    } yield d
-
-  def temp2[F[_]: Files](
-      parent: Path,
-      prefix1: String,
-      prefix2: String
-  ): Resource[F, (Path, Path)] =
-    for {
-      _ <- Resource.eval(Files[F].createDirectories(parent))
-      a <- mkTemp(parent, prefix1)
-      b <- mkTemp(parent, prefix2)
-    } yield (a, b)
-
-  def createTemp[F[_]: Files: Monad](
-      parent: Path,
-      prefix: String
-  ): F[Path] =
-    for {
-      _ <- Files[F].createDirectories(parent)
-      d <- mkTemp_(parent, prefix)
-    } yield d
-
-  private def mkTemp[F[_]: Files](parent: Path, prefix: String): Resource[F, Path] =
-    Files[F]
-      .tempDirectory(
-        parent.some,
-        prefix,
-        PosixPermissions.fromOctal("777")
-      )
-
-  private def mkTemp_[F[_]: Files](parent: Path, prefix: String): F[Path] =
-    Files[F]
-      .createTempDirectory(
-        parent.some,
-        prefix,
-        PosixPermissions.fromOctal("777")
-      )
-}

modules/addonlib/src/main/scala/docspell/addons/InputEnv.scala

@@ -10,6 +10,7 @@ import cats.effect.Resource
 import fs2.io.file.{Files, Path}
 
 import docspell.common.exec.Env
+import docspell.common.util.Directory
 
 case class InputEnv(
     addons: List[AddonRef],

modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala

@@ -13,7 +13,7 @@ import fs2.io.file.{Files, Path, PosixPermissions}
 
 import docspell.addons.out.AddonOutput
 import docspell.common.LenientUri
-import docspell.files.Zip
+import docspell.common.util.Zip
 
 import io.circe.syntax._
 
@@ -59,9 +59,9 @@ object AddonGenerator {
   private def createZip(dir: Path, files: List[Path]) =
     Stream
       .emits(files)
-      .map(f => (f.fileName.toString, Files[IO].readAll(f)))
+      .map(f => (f.fileName.toString, f))
       .covary[IO]
-      .through(Zip.zip[IO](logger, 8192))
+      .through(Zip[IO](logger.some).zipFiles())
       .through(Files[IO].writeAll(dir / "addon.zip"))
       .compile
       .drain

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -7,9 +7,10 @@
 package docspell.addons
 
 import cats.effect._
+import cats.syntax.all._
 
 import docspell.common.Glob
-import docspell.files.Zip
+import docspell.common.util.{Directory, Zip}
 import docspell.logging.TestLoggingConfig
 
 import munit._
@@ -26,8 +27,8 @@ class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures
     for {
       _ <- dummyAddonUrl
         .readURL[IO](8192)
-        .through(Zip.unzip(8192, Glob.all))
-        .through(Zip.saveTo(logger, dir, moveUp = true))
+        .through(Zip[IO]().unzip(8192, Glob.all, dir.some))
+        .evalTap(_ => Directory.unwrapSingle(logger, dir))
         .compile
         .drain
       meta <- AddonMeta.findInDirectory[IO](dir)