diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala
index 630857fd..c3b4a660 100644
--- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala
@@ -65,7 +65,7 @@ object FileCopyTask {
if (result.success)
s"Successfully copied $allGood files to ${result.counter.size} stores."
else
- s"Copying files failed for ${failed} files! ${allGood} were copied successfully."
+ s"Copying files failed for ${failed} files! $allGood were copied successfully."
}
}
@@ -122,8 +122,8 @@ object FileCopyTask {
case None =>
CopyResult.noSourceImpl.pure[F]
- case Some((src, srcMeta)) =>
- to.traverse(FileRepository.getDelegate).map(_.map(_._1)) match {
+ case Some(src) =>
+ to.traverse(FileRepository.getDelegate) match {
case None =>
CopyResult.noTargetImpl.pure[F]
@@ -135,7 +135,7 @@ object FileCopyTask {
}
def copyTo(to: BinaryStore[F]) =
- CopyTool.copyAll[F](log, src, srcMeta, to, 50, maxConcurrent)
+ CopyTool.copyAll[F](log, src, to, 50, maxConcurrent)
logger.info(s"Start copying ${from.config} -> ${to.map(_.config)}") *>
targets.traverse(copyTo).map(CopyResult.success)
diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala
index 020fc9f9..653dab0f 100644
--- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala
+++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala
@@ -13,8 +13,8 @@ import scribe.output.format.OutputFormat
import scribe.writer._
final case class JsonWriter(writer: Writer, compact: Boolean = true) extends Writer {
- override def write[M](
- record: LogRecord[M],
+ override def write(
+ record: LogRecord,
output: LogOutput,
outputFormat: OutputFormat
): Unit = {
diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala
index f9f4db12..01ef9adb 100644
--- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala
+++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala
@@ -14,8 +14,8 @@ import scribe.writer._
// https://brandur.org/logfmt
final case class LogfmtWriter(writer: Writer) extends Writer {
- override def write[M](
- record: LogRecord[M],
+ override def write(
+ record: LogRecord,
output: LogOutput,
outputFormat: OutputFormat
): Unit = {
diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala
index 68123a5f..165b2333 100644
--- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala
+++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala
@@ -38,18 +38,23 @@ private[impl] case class Record(
private[impl] object Record {
- def fromLogRecord[M](record: LogRecord[M]): Record = {
+ def fromLogRecord(record: LogRecord): Record = {
val l = record.timeStamp
- val traces = record.additionalMessages.collect {
+ val traces = record.messages.collect {
case message: Message[_] if message.value.isInstanceOf[Throwable] =>
throwable2Trace(message.value.asInstanceOf[Throwable])
}
- val additionalMessages = record.additionalMessages.map(_.logOutput.plainText)
+ val (firstMessage, additionalMessages) = record.messages match {
+ case h :: rest =>
+ (h.logOutput.plainText, rest.map(_.logOutput.plainText))
+ case Nil =>
+ ("", Nil)
+ }
Record(
level = record.level.name,
levelValue = record.levelValue,
- message = record.logOutput.plainText,
+ message = firstMessage,
additionalMessages = additionalMessages,
fileName = record.fileName,
className = record.className,
diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala
index d547225c..143fe7fc 100644
--- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala
+++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala
@@ -12,7 +12,7 @@ import cats.effect.Sync
import docspell.logging.{Level, LogEvent, Logger}
import scribe.LoggerSupport
-import scribe.message.{LoggableMessage, Message}
+import scribe.message.LoggableMessage
private[logging] object ScribeWrapper {
final class ImplUnsafe(log: scribe.Logger) extends Logger[Id] {
@@ -41,10 +41,10 @@ private[logging] object ScribeWrapper {
private[this] def convert(ev: LogEvent) = {
val level = convertLevel(ev.level)
val additional: List[LoggableMessage] = ev.additional.map {
- case Right(ex) => Message.static(ex)
- case Left(msg) => Message.static(msg)
+ case Right(ex) => LoggableMessage.throwable2Message(ex)
+ case Left(msg) => LoggableMessage.string2Message(msg)
}.toList
- LoggerSupport(level, ev.msg(), additional, ev.pkg, ev.fileName, ev.name, ev.line)
+ LoggerSupport(level, ev.msg() :: additional, ev.pkg, ev.fileName, ev.name, ev.line)
.copy(data = ev.data.toDeferred)
}
}
diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala
index adb21536..dc2b05db 100644
--- a/modules/store/src/main/scala/docspell/store/Store.scala
+++ b/modules/store/src/main/scala/docspell/store/Store.scala
@@ -60,7 +60,7 @@ object Store {
ds.setDriverClassName(jdbc.dbms.driverClass)
}
xa = HikariTransactor(ds, connectEC)
- fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
+ fr = FileRepository(xa, ds, fileRepoConfig, true)
st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa)
_ <- Resource.eval(st.migrate)
} yield st
diff --git a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala
index 127788b2..54153bd1 100644
--- a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala
+++ b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala
@@ -18,71 +18,52 @@ import binny._
import doobie._
import doobie.implicits._
-private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] {
- def findMeta(id: BinaryId): OptionT[F, RFileMeta]
+trait AttributeStore[F[_]] {
+ def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit]
+
+ def deleteAttr(key: FileKey): F[Boolean]
+
+ def findMeta(key: FileKey): OptionT[F, RFileMeta]
}
private[file] object AttributeStore {
def empty[F[_]: Applicative]: AttributeStore[F] =
new AttributeStore[F] {
- val delegate = BinaryAttributeStore.empty[F]
+ override def saveAttr(key: FileKey, attrs: BinaryAttributes) = ().pure[F]
- def findMeta(id: BinaryId) =
- OptionT.none
+ override def deleteAttr(key: FileKey) = false.pure[F]
- def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) =
- delegate.saveAttr(id, attrs)
-
- def deleteAttr(id: BinaryId) =
- delegate.deleteAttr(id)
-
- def findAttr(id: BinaryId) =
- delegate.findAttr(id)
+ override def findMeta(key: FileKey) = OptionT.none[F, RFileMeta]
}
def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] =
new Impl[F](xa)
final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] {
- def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] =
+ def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] =
for {
now <- Timestamp.current[F]
- a <- attrs
- fileKey <- makeFileKey(id)
fm = RFileMeta(
- fileKey,
+ key,
now,
- MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream),
- ByteSize(a.length),
- a.sha256
+ MimeType.parse(attrs.contentType.contentType).getOrElse(MimeType.octetStream),
+ ByteSize(attrs.length),
+ attrs.sha256
)
_ <- RFileMeta.insert(fm).transact(xa)
} yield ()
- def deleteAttr(id: BinaryId): F[Boolean] =
- makeFileKey(id).flatMap(fileKey =>
- RFileMeta.delete(fileKey).transact(xa).map(_ > 0)
- )
+ def deleteAttr(key: FileKey): F[Boolean] =
+ RFileMeta.delete(key).transact(xa).map(_ > 0)
- def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] =
- findMeta(id).map(fm =>
- BinaryAttributes(
- fm.checksum,
- SimpleContentType(fm.mimetype.asString),
- fm.length.bytes
- )
- )
+ def findMeta(key: FileKey): OptionT[F, RFileMeta] =
+ OptionT(RFileMeta.findById(key).transact(xa))
- def findMeta(id: BinaryId): OptionT[F, RFileMeta] =
- OptionT(
- makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa))
- )
-
- private def makeFileKey(binaryId: BinaryId): F[FileKey] =
- Sync[F]
- .pure(
- BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))
- )
- .rethrow
+// private def makeFileKey(binaryId: BinaryId): F[FileKey] =
+// Sync[F]
+// .pure(
+// BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))
+// )
+// .rethrow
}
}
diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala
index e87d9d3d..15eb4892 100644
--- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala
+++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala
@@ -16,7 +16,7 @@ import docspell.files.TikaMimetype
import docspell.logging.Logger
import binny._
-import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping}
+import binny.fs._
import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping}
import scodec.bits.ByteVector
@@ -95,7 +95,6 @@ object BinnyUtils {
def binaryStore[F[_]: Async](
cfg: FileRepositoryConfig,
- attrStore: AttributeStore[F],
ds: DataSource,
logger: Logger[F]
): BinaryStore[F] =
@@ -103,7 +102,7 @@ object BinnyUtils {
case FileRepositoryConfig.Database(chunkSize) =>
val jdbcConfig =
JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect)
- GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore)
+ GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig)
case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) =>
val keyMapping = S3KeyMapping.constant(bucket)
@@ -111,16 +110,16 @@ object BinnyUtils {
.default(endpoint, accessKey, secretKey, keyMapping)
.copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect)
- MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger))
+ MinioBinaryStore[F](minioCfg, LoggerAdapter(logger))
case FileRepositoryConfig.Directory(path, chunkSize) =>
val fsConfig = FsStoreConfig(
path,
BinnyUtils.TikaContentTypeDetect,
- FsStoreConfig.OverwriteMode.Fail,
+ OverwriteMode.Fail,
BinnyUtils.pathMapping,
chunkSize
)
- FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore)
+ FsBinaryStore[F](fsConfig, LoggerAdapter(logger))
}
}
diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala
index 48b30b71..53d96815 100644
--- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala
+++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala
@@ -13,7 +13,7 @@ import fs2._
import docspell.common._
-import binny.{BinaryAttributeStore, BinaryId, BinaryStore}
+import binny.{BinaryId, BinaryStore}
import doobie.Transactor
trait FileRepository[F[_]] {
@@ -45,17 +45,16 @@ object FileRepository {
else AttributeStore.empty[F]
val log = docspell.logging.getLogger[F]
val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId
- val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log)
-
+ val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, ds, log)
new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun)
}
def getDelegate[F[_]](
repo: FileRepository[F]
- ): Option[(BinaryStore[F], BinaryAttributeStore[F])] =
+ ): Option[BinaryStore[F]] =
repo match {
case n: FileRepositoryImpl[F] =>
- Some((n.bs, n.attrStore))
+ Some(n.bs)
case _ =>
None
diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala
index e605d79c..d612d7ed 100644
--- a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala
+++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala
@@ -30,14 +30,14 @@ final class FileRepositoryImpl[F[_]: Sync](
def findMeta(key: FileKey): F[Option[FileMetadata]] =
attrStore
- .findMeta(keyFun(key))
+ .findMeta(key)
.map(rfm =>
FileMetadata(rfm.id, rfm.created, rfm.mimetype, rfm.length, rfm.checksum)
)
.value
def delete(key: FileKey): F[Unit] =
- bs.delete(keyFun(key))
+ bs.delete(keyFun(key)) *> attrStore.deleteAttr(key).void
def save(
collective: Ident,
@@ -48,9 +48,15 @@ final class FileRepositoryImpl[F[_]: Sync](
in =>
Stream
.eval(randomKey(collective, category))
- .flatMap(fkey =>
- in.through(bs.insertWith(keyFun(fkey), fhint)) ++ Stream.emit(fkey)
- )
+ .flatMap(fkey => in.through(bs.insertWith(keyFun(fkey))) ++ Stream.emit(fkey))
+ .evalTap { key =>
+ val bid = keyFun(key)
+ bs.computeAttr(bid, fhint)
+ .run(AttributeName.all)
+ .semiflatMap(attr => attrStore.saveAttr(key, attr))
+ .value
+ .void
+ }
}
def randomKey(
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 27d0d9e6..25b405e3 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -7,7 +7,7 @@ object Dependencies {
val BcryptVersion = "0.4"
val BetterMonadicForVersion = "0.3.1"
- val BinnyVersion = "0.4.0"
+ val BinnyVersion = "0.6.0"
val CalevVersion = "0.6.3"
val CatsVersion = "2.7.0"
val CatsEffectVersion = "3.3.12"
@@ -43,7 +43,7 @@ object Dependencies {
val PureConfigVersion = "0.17.1"
val ScalaJavaTimeVersion = "2.4.0"
val ScodecBitsVersion = "1.1.31"
- val ScribeVersion = "3.8.3"
+ val ScribeVersion = "3.9.0"
val Slf4jVersion = "1.7.36"
val SourcecodeVersion = "0.2.8"
val StanfordNlpVersion = "4.4.0"