TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-05 02:49:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1610 from eikek/update-deps

Browse Source 
Update deps
This commit is contained in:
mergify[bot] 2022-06-27 11:43:12 +00:00 committed by GitHub
commit ce497b05ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 68 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala
View File

@ -65,7 +65,7 @@ object FileCopyTask {
if (result.success)
s"Successfully copied $allGood files to ${result.counter.size} stores."
else
s"Copying files failed for ${failed} files! ${allGood} were copied successfully."
s"Copying files failed for ${failed} files! $allGood were copied successfully."
}
}
@ -122,8 +122,8 @@ object FileCopyTask {
case None =>
CopyResult.noSourceImpl.pure[F]
case Some((src, srcMeta)) =>
to.traverse(FileRepository.getDelegate).map(_.map(_._1)) match {
case Some(src) =>
to.traverse(FileRepository.getDelegate) match {
case None =>
CopyResult.noTargetImpl.pure[F]
@ -135,7 +135,7 @@ object FileCopyTask {
}
def copyTo(to: BinaryStore[F]) =
CopyTool.copyAll[F](log, src, srcMeta, to, 50, maxConcurrent)
CopyTool.copyAll[F](log, src, to, 50, maxConcurrent)
logger.info(s"Start copying ${from.config} -> ${to.map(_.config)}") *>
targets.traverse(copyTo).map(CopyResult.success)

4
modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala
View File

@ -13,8 +13,8 @@ import scribe.output.format.OutputFormat
import scribe.writer._
final case class JsonWriter(writer: Writer, compact: Boolean = true) extends Writer {
override def write[M](
record: LogRecord[M],
override def write(
record: LogRecord,
output: LogOutput,
outputFormat: OutputFormat
): Unit = {

4
modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala
View File

@ -14,8 +14,8 @@ import scribe.writer._
// https://brandur.org/logfmt
final case class LogfmtWriter(writer: Writer) extends Writer {
override def write[M](
record: LogRecord[M],
override def write(
record: LogRecord,
output: LogOutput,
outputFormat: OutputFormat
): Unit = {

13
modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala
View File

@ -38,18 +38,23 @@ private[impl] case class Record(
private[impl] object Record {
def fromLogRecord[M](record: LogRecord[M]): Record = {
def fromLogRecord(record: LogRecord): Record = {
val l = record.timeStamp
val traces = record.additionalMessages.collect {
val traces = record.messages.collect {
case message: Message[_] if message.value.isInstanceOf[Throwable] =>
throwable2Trace(message.value.asInstanceOf[Throwable])
}
val additionalMessages = record.additionalMessages.map(_.logOutput.plainText)
val (firstMessage, additionalMessages) = record.messages match {
case h :: rest =>
(h.logOutput.plainText, rest.map(_.logOutput.plainText))
case Nil =>
("", Nil)
}
Record(
level = record.level.name,
levelValue = record.levelValue,
message = record.logOutput.plainText,
message = firstMessage,
additionalMessages = additionalMessages,
fileName = record.fileName,
className = record.className,

8
modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala
View File

@ -12,7 +12,7 @@ import cats.effect.Sync
import docspell.logging.{Level, LogEvent, Logger}
import scribe.LoggerSupport
import scribe.message.{LoggableMessage, Message}
import scribe.message.LoggableMessage
private[logging] object ScribeWrapper {
final class ImplUnsafe(log: scribe.Logger) extends Logger[Id] {
@ -41,10 +41,10 @@ private[logging] object ScribeWrapper {
private[this] def convert(ev: LogEvent) = {
val level = convertLevel(ev.level)
val additional: List[LoggableMessage] = ev.additional.map {
case Right(ex) => Message.static(ex)
case Left(msg) => Message.static(msg)
case Right(ex) => LoggableMessage.throwable2Message(ex)
case Left(msg) => LoggableMessage.string2Message(msg)
}.toList
LoggerSupport(level, ev.msg(), additional, ev.pkg, ev.fileName, ev.name, ev.line)
LoggerSupport(level, ev.msg() :: additional, ev.pkg, ev.fileName, ev.name, ev.line)
.copy(data = ev.data.toDeferred)
}
}

2
modules/store/src/main/scala/docspell/store/Store.scala
View File

@ -60,7 +60,7 @@ object Store {
ds.setDriverClassName(jdbc.dbms.driverClass)
}
xa = HikariTransactor(ds, connectEC)
fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
fr = FileRepository(xa, ds, fileRepoConfig, true)
st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa)
_ <- Resource.eval(st.migrate)
} yield st

67
modules/store/src/main/scala/docspell/store/file/AttributeStore.scala
View File

@ -18,71 +18,52 @@ import binny._
import doobie._
import doobie.implicits._
private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] {
def findMeta(id: BinaryId): OptionT[F, RFileMeta]
trait AttributeStore[F[_]] {
def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit]
def deleteAttr(key: FileKey): F[Boolean]
def findMeta(key: FileKey): OptionT[F, RFileMeta]
}
private[file] object AttributeStore {
def empty[F[_]: Applicative]: AttributeStore[F] =
new AttributeStore[F] {
val delegate = BinaryAttributeStore.empty[F]
override def saveAttr(key: FileKey, attrs: BinaryAttributes) = ().pure[F]
def findMeta(id: BinaryId) =
OptionT.none
override def deleteAttr(key: FileKey) = false.pure[F]
def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) =
delegate.saveAttr(id, attrs)
def deleteAttr(id: BinaryId) =
delegate.deleteAttr(id)
def findAttr(id: BinaryId) =
delegate.findAttr(id)
override def findMeta(key: FileKey) = OptionT.none[F, RFileMeta]
}
def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] =
new Impl[F](xa)
final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] {
def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] =
def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] =
for {
now <- Timestamp.current[F]
a <- attrs
fileKey <- makeFileKey(id)
fm = RFileMeta(
fileKey,
key,
now,
MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream),
ByteSize(a.length),
a.sha256
MimeType.parse(attrs.contentType.contentType).getOrElse(MimeType.octetStream),
ByteSize(attrs.length),
attrs.sha256
)
_ <- RFileMeta.insert(fm).transact(xa)
} yield ()
def deleteAttr(id: BinaryId): F[Boolean] =
makeFileKey(id).flatMap(fileKey =>
RFileMeta.delete(fileKey).transact(xa).map(_ > 0)
)
def deleteAttr(key: FileKey): F[Boolean] =
RFileMeta.delete(key).transact(xa).map(_ > 0)
def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] =
findMeta(id).map(fm =>
BinaryAttributes(
fm.checksum,
SimpleContentType(fm.mimetype.asString),
fm.length.bytes
)
)
def findMeta(key: FileKey): OptionT[F, RFileMeta] =
OptionT(RFileMeta.findById(key).transact(xa))
def findMeta(id: BinaryId): OptionT[F, RFileMeta] =
OptionT(
makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa))
)
private def makeFileKey(binaryId: BinaryId): F[FileKey] =
Sync[F]
.pure(
BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))
)
.rethrow
// private def makeFileKey(binaryId: BinaryId): F[FileKey] =
// Sync[F]
// .pure(
// BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))
// )
// .rethrow
}
}

11
modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala
View File

@ -16,7 +16,7 @@ import docspell.files.TikaMimetype
import docspell.logging.Logger
import binny._
import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping}
import binny.fs._
import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping}
import scodec.bits.ByteVector
@ -95,7 +95,6 @@ object BinnyUtils {
def binaryStore[F[_]: Async](
cfg: FileRepositoryConfig,
attrStore: AttributeStore[F],
ds: DataSource,
logger: Logger[F]
): BinaryStore[F] =
@ -103,7 +102,7 @@ object BinnyUtils {
case FileRepositoryConfig.Database(chunkSize) =>
val jdbcConfig =
JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect)
GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore)
GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig)
case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) =>
val keyMapping = S3KeyMapping.constant(bucket)
@ -111,16 +110,16 @@ object BinnyUtils {
.default(endpoint, accessKey, secretKey, keyMapping)
.copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect)
MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger))
MinioBinaryStore[F](minioCfg, LoggerAdapter(logger))
case FileRepositoryConfig.Directory(path, chunkSize) =>
val fsConfig = FsStoreConfig(
path,
BinnyUtils.TikaContentTypeDetect,
FsStoreConfig.OverwriteMode.Fail,
OverwriteMode.Fail,
BinnyUtils.pathMapping,
chunkSize
)
FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore)
FsBinaryStore[F](fsConfig, LoggerAdapter(logger))
}
}

9
modules/store/src/main/scala/docspell/store/file/FileRepository.scala
View File

@ -13,7 +13,7 @@ import fs2._
import docspell.common._
import binny.{BinaryAttributeStore, BinaryId, BinaryStore}
import binny.{BinaryId, BinaryStore}
import doobie.Transactor
trait FileRepository[F[_]] {
@ -45,17 +45,16 @@ object FileRepository {
else AttributeStore.empty[F]
val log = docspell.logging.getLogger[F]
val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId
val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log)
val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, ds, log)
new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun)
}
def getDelegate[F[_]](
repo: FileRepository[F]
): Option[(BinaryStore[F], BinaryAttributeStore[F])] =
): Option[BinaryStore[F]] =
repo match {
case n: FileRepositoryImpl[F] =>
Some((n.bs, n.attrStore))
Some(n.bs)
case _ =>
None

16
modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala
View File

@ -30,14 +30,14 @@ final class FileRepositoryImpl[F[_]: Sync](
def findMeta(key: FileKey): F[Option[FileMetadata]] =
attrStore
.findMeta(keyFun(key))
.findMeta(key)
.map(rfm =>
FileMetadata(rfm.id, rfm.created, rfm.mimetype, rfm.length, rfm.checksum)
)
.value
def delete(key: FileKey): F[Unit] =
bs.delete(keyFun(key))
bs.delete(keyFun(key)) *> attrStore.deleteAttr(key).void
def save(
collective: Ident,
@ -48,9 +48,15 @@ final class FileRepositoryImpl[F[_]: Sync](
in =>
Stream
.eval(randomKey(collective, category))
.flatMap(fkey =>
in.through(bs.insertWith(keyFun(fkey), fhint)) ++ Stream.emit(fkey)
)
.flatMap(fkey => in.through(bs.insertWith(keyFun(fkey))) ++ Stream.emit(fkey))
.evalTap { key =>
val bid = keyFun(key)
bs.computeAttr(bid, fhint)
.run(AttributeName.all)
.semiflatMap(attr => attrStore.saveAttr(key, attr))
.value
.void
}
}
def randomKey(

4
project/Dependencies.scala
View File

@ -7,7 +7,7 @@ object Dependencies {
val BcryptVersion = "0.4"
val BetterMonadicForVersion = "0.3.1"
val BinnyVersion = "0.4.0"
val BinnyVersion = "0.6.0"
val CalevVersion = "0.6.3"
val CatsVersion = "2.7.0"
val CatsEffectVersion = "3.3.12"
@ -43,7 +43,7 @@ object Dependencies {
val PureConfigVersion = "0.17.1"
val ScalaJavaTimeVersion = "2.4.0"
val ScodecBitsVersion = "1.1.31"
val ScribeVersion = "3.8.3"
val ScribeVersion = "3.9.0"
val Slf4jVersion = "1.7.36"
val SourcecodeVersion = "0.2.8"
val StanfordNlpVersion = "4.4.0"