TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade code base to CE3

Browse Source
This commit is contained in:
eikek
2021-06-21 21:33:54 +02:00
parent 903ec26e54
commit bd791b4593
146 changed files with 638 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
modules/common/src/main/scala/docspell/common/File.scala
View File

@ -1,47 +1,48 @@
package docspell.common
import java.io.IOException
import java.nio.charset.StandardCharsets
import java.nio.file._
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.{Files => JFiles, _}
import java.util.concurrent.atomic.AtomicInteger
import scala.jdk.CollectionConverters._
import cats.effect._
import cats.implicits._
import fs2.Stream
import fs2.io.file.Files
import fs2.{Chunk, Stream}
import docspell.common.syntax.all._
import io.circe.Decoder
import scodec.bits.ByteVector
//TODO use io.fs2.files.Files api
object File {
def mkDir[F[_]: Sync](dir: Path): F[Path] =
Sync[F].delay(Files.createDirectories(dir))
Sync[F].blocking(JFiles.createDirectories(dir))
def mkTempDir[F[_]: Sync](parent: Path, prefix: String): F[Path] =
mkDir(parent).map(p => Files.createTempDirectory(p, prefix))
mkDir(parent).map(p => JFiles.createTempDirectory(p, prefix))
def mkTempFile[F[_]: Sync](
parent: Path,
prefix: String,
suffix: Option[String] = None
): F[Path] =
mkDir(parent).map(p => Files.createTempFile(p, prefix, suffix.orNull))
mkDir(parent).map(p => JFiles.createTempFile(p, prefix, suffix.orNull))
def deleteDirectory[F[_]: Sync](dir: Path): F[Int] =
Sync[F].delay {
val count = new AtomicInteger(0)
Files.walkFileTree(
JFiles.walkFileTree(
dir,
new SimpleFileVisitor[Path]() {
override def visitFile(
file: Path,
attrs: BasicFileAttributes
): FileVisitResult = {
Files.deleteIfExists(file)
JFiles.deleteIfExists(file)
count.incrementAndGet()
FileVisitResult.CONTINUE
}
@ -49,7 +50,7 @@ object File {
Option(e) match {
case Some(ex) => throw ex
case None =>
Files.deleteIfExists(dir)
JFiles.deleteIfExists(dir)
FileVisitResult.CONTINUE
}
}
@ -58,47 +59,57 @@ object File {
}
def exists[F[_]: Sync](file: Path): F[Boolean] =
Sync[F].delay(Files.exists(file))
Sync[F].delay(JFiles.exists(file))
def size[F[_]: Sync](file: Path): F[Long] =
Sync[F].delay(Files.size(file))
Sync[F].delay(JFiles.size(file))
def existsNonEmpty[F[_]: Sync](file: Path, minSize: Long = 0): F[Boolean] =
Sync[F].delay(Files.exists(file) && Files.size(file) > minSize)
Sync[F].delay(JFiles.exists(file) && JFiles.size(file) > minSize)
def deleteFile[F[_]: Sync](file: Path): F[Unit] =
Sync[F].delay(Files.deleteIfExists(file)).map(_ => ())
Sync[F].delay(JFiles.deleteIfExists(file)).map(_ => ())
def delete[F[_]: Sync](path: Path): F[Int] =
if (Files.isDirectory(path)) deleteDirectory(path)
if (JFiles.isDirectory(path)) deleteDirectory(path)
else deleteFile(path).map(_ => 1)
def withTempDir[F[_]: Sync](parent: Path, prefix: String): Resource[F, Path] =
Resource.make(mkTempDir(parent, prefix))(p => delete(p).map(_ => ()))
def listFiles[F[_]: Sync](pred: Path => Boolean, dir: Path): F[List[Path]] =
def listJFiles[F[_]: Sync](pred: Path => Boolean, dir: Path): F[List[Path]] =
Sync[F].delay {
val javaList =
Files.list(dir).filter(p => pred(p)).collect(java.util.stream.Collectors.toList())
JFiles
.list(dir)
.filter(p => pred(p))
.collect(java.util.stream.Collectors.toList())
javaList.asScala.toList.sortBy(_.getFileName.toString)
}
def readAll[F[_]: Sync: ContextShift](
def readAll[F[_]: Files](
file: Path,
blocker: Blocker,
chunkSize: Int
): Stream[F, Byte] =
fs2.io.file.readAll(file, blocker, chunkSize)
Files[F].readAll(file, chunkSize)
def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] =
readAll[F](file, blocker, 8192).through(fs2.text.utf8Decode).compile.foldMonoid
def readText[F[_]: Files: Concurrent](file: Path): F[String] =
readAll[F](file, 8192).through(fs2.text.utf8Decode).compile.foldMonoid
def writeString[F[_]: Sync](file: Path, content: String): F[Path] =
Sync[F].delay(Files.write(file, content.getBytes(StandardCharsets.UTF_8)))
def writeString[F[_]: Files: Concurrent](file: Path, content: String): F[Path] =
ByteVector.encodeUtf8(content) match {
case Right(bv) =>
Stream
.chunk(Chunk.byteVector(bv))
.through(Files[F].writeAll(file))
.compile
.drain
.map(_ => file)
case Left(ex) =>
Concurrent[F].raiseError(ex)
}
def readJson[F[_]: Sync: ContextShift, A](file: Path, blocker: Blocker)(implicit
d: Decoder[A]
): F[A] =
readText[F](file, blocker).map(_.parseJsonAs[A]).rethrow
def readJson[F[_]: Async, A](file: Path)(implicit d: Decoder[A]): F[A] =
readText[F](file).map(_.parseJsonAs[A]).rethrow
}

13
modules/common/src/main/scala/docspell/common/LenientUri.scala
View File

@ -6,7 +6,7 @@ import java.net.URLEncoder
import cats.data.NonEmptyList
import cats.effect.Resource
import cats.effect.{Blocker, ContextShift, Sync}
import cats.effect._
import cats.implicits._
import fs2.Stream
@ -66,20 +66,17 @@ case class LenientUri(
)
}
def readURL[F[_]: Sync: ContextShift](
chunkSize: Int,
blocker: Blocker
): Stream[F, Byte] =
def readURL[F[_]: Sync](chunkSize: Int): Stream[F, Byte] =
Stream
.emit(Either.catchNonFatal(new URL(asString)))
.covary[F]
.rethrow
.flatMap(url =>
fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, blocker, true)
fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, true)
)
def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
readURL[F](chunkSize, blocker).through(fs2.text.utf8Decode).compile.foldMonoid
def readText[F[_]: Sync](chunkSize: Int): F[String] =
readURL[F](chunkSize).through(fs2.text.utf8Decode).compile.foldMonoid
def host: Option[String] =
authority.map(a =>

3
modules/common/src/main/scala/docspell/common/Pools.scala
View File

@ -2,13 +2,10 @@ package docspell.common
import scala.concurrent.ExecutionContext
import cats.effect._
/** Captures thread pools to use in an application.
*/
case class Pools(
connectEC: ExecutionContext,
httpClientEC: ExecutionContext,
blocker: Blocker,
restEC: ExecutionContext
)

36
modules/common/src/main/scala/docspell/common/SystemCommand.scala
View File

@ -7,7 +7,7 @@ import java.util.concurrent.TimeUnit
import scala.jdk.CollectionConverters._
import cats.effect.{Blocker, ContextShift, Sync}
import cats.effect._
import cats.implicits._
import fs2.{Stream, io, text}
@ -34,9 +34,8 @@ object SystemCommand {
final case class Result(rc: Int, stdout: String, stderr: String)
def exec[F[_]: Sync: ContextShift](
def exec[F[_]: Sync](
cmd: Config,
blocker: Blocker,
logger: Logger[F],
wd: Option[Path] = None,
stdin: Stream[F, Byte] = Stream.empty
@ -44,8 +43,8 @@ object SystemCommand {
startProcess(cmd, wd, logger, stdin) { proc =>
Stream.eval {
for {
_ <- writeToProcess(stdin, proc, blocker)
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
_ <- writeToProcess(stdin, proc)
term <- Sync[F].blocking(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
_ <-
if (term)
logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
@ -55,23 +54,22 @@ object SystemCommand {
)
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
out <-
if (term) inputStreamToString(proc.getInputStream, blocker)
if (term) inputStreamToString(proc.getInputStream)
else Sync[F].pure("")
err <-
if (term) inputStreamToString(proc.getErrorStream, blocker)
if (term) inputStreamToString(proc.getErrorStream)
else Sync[F].pure("")
} yield Result(proc.exitValue, out, err)
}
}
def execSuccess[F[_]: Sync: ContextShift](
def execSuccess[F[_]: Sync](
cmd: Config,
blocker: Blocker,
logger: Logger[F],
wd: Option[Path] = None,
stdin: Stream[F, Byte] = Stream.empty
): Stream[F, Result] =
exec(cmd, blocker, logger, wd, stdin).flatMap { r =>
exec(cmd, logger, wd, stdin).flatMap { r =>
if (r.rc != 0)
Stream.raiseError[F](
new Exception(
@ -92,7 +90,7 @@ object SystemCommand {
val log = logger.debug(s"Running external command: ${cmd.cmdString}")
val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
val proc = log *> hasStdin.flatMap(flag =>
Sync[F].delay {
Sync[F].blocking {
val pb = new ProcessBuilder(cmd.toCmd.asJava)
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
.redirectError(Redirect.PIPE)
@ -109,11 +107,8 @@ object SystemCommand {
.flatMap(f)
}
private def inputStreamToString[F[_]: Sync: ContextShift](
in: InputStream,
blocker: Blocker
): F[String] =
io.readInputStream(Sync[F].pure(in), 16 * 1024, blocker, closeAfterUse = false)
private def inputStreamToString[F[_]: Sync](in: InputStream): F[String] =
io.readInputStream(Sync[F].pure(in), 16 * 1024, closeAfterUse = false)
.through(text.utf8Decode)
.chunks
.map(_.toVector.mkString)
@ -122,18 +117,17 @@ object SystemCommand {
.last
.map(_.getOrElse(""))
private def writeToProcess[F[_]: Sync: ContextShift](
private def writeToProcess[F[_]: Sync](
data: Stream[F, Byte],
proc: Process,
blocker: Blocker
proc: Process
): F[Unit] =
data
.through(io.writeOutputStream(Sync[F].delay(proc.getOutputStream), blocker))
.through(io.writeOutputStream(Sync[F].blocking(proc.getOutputStream)))
.compile
.drain
private def timeoutError[F[_]: Sync](proc: Process, cmd: Config): F[Unit] =
Sync[F].delay(proc.destroyForcibly()).attempt *> {
Sync[F].blocking(proc.destroyForcibly()).attempt *> {
Sync[F].raiseError(
new Exception(
s"Command `${cmd.cmdString}` timed out (${cmd.timeout.formatExact})"