From bca2ea28e3743c43aaaf19042854f87140af0728 Mon Sep 17 00:00:00 2001 From: eikek Date: Mon, 27 Jun 2022 13:23:48 +0200 Subject: [PATCH 1/2] Update binny to 0.6.0 --- .../docspell/joex/filecopy/FileCopyTask.scala | 8 +-- .../src/main/scala/docspell/store/Store.scala | 2 +- .../docspell/store/file/AttributeStore.scala | 67 +++++++------------ .../docspell/store/file/BinnyUtils.scala | 11 ++- .../docspell/store/file/FileRepository.scala | 9 ++- .../store/file/FileRepositoryImpl.scala | 16 +++-- project/Dependencies.scala | 2 +- 7 files changed, 50 insertions(+), 65 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 630857fd..c3b4a660 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -65,7 +65,7 @@ object FileCopyTask { if (result.success) s"Successfully copied $allGood files to ${result.counter.size} stores." else - s"Copying files failed for ${failed} files! ${allGood} were copied successfully." + s"Copying files failed for ${failed} files! $allGood were copied successfully." } } @@ -122,8 +122,8 @@ object FileCopyTask { case None => CopyResult.noSourceImpl.pure[F] - case Some((src, srcMeta)) => - to.traverse(FileRepository.getDelegate).map(_.map(_._1)) match { + case Some(src) => + to.traverse(FileRepository.getDelegate) match { case None => CopyResult.noTargetImpl.pure[F] @@ -135,7 +135,7 @@ object FileCopyTask { } def copyTo(to: BinaryStore[F]) = - CopyTool.copyAll[F](log, src, srcMeta, to, 50, maxConcurrent) + CopyTool.copyAll[F](log, src, to, 50, maxConcurrent) logger.info(s"Start copying ${from.config} -> ${to.map(_.config)}") *> targets.traverse(copyTo).map(CopyResult.success) diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index adb21536..dc2b05db 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -60,7 +60,7 @@ object Store { ds.setDriverClassName(jdbc.dbms.driverClass) } xa = HikariTransactor(ds, connectEC) - fr = FileRepository.apply(xa, ds, fileRepoConfig, true) + fr = FileRepository(xa, ds, fileRepoConfig, true) st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa) _ <- Resource.eval(st.migrate) } yield st diff --git a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala index 127788b2..54153bd1 100644 --- a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala +++ b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala @@ -18,71 +18,52 @@ import binny._ import doobie._ import doobie.implicits._ -private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] { - def findMeta(id: BinaryId): OptionT[F, RFileMeta] +trait AttributeStore[F[_]] { + def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] + + def deleteAttr(key: FileKey): F[Boolean] + + def findMeta(key: FileKey): OptionT[F, RFileMeta] } private[file] object AttributeStore { def empty[F[_]: Applicative]: AttributeStore[F] = new AttributeStore[F] { - val delegate = BinaryAttributeStore.empty[F] + override def saveAttr(key: FileKey, attrs: BinaryAttributes) = ().pure[F] - def findMeta(id: BinaryId) = - OptionT.none + override def deleteAttr(key: FileKey) = false.pure[F] - def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) = - delegate.saveAttr(id, attrs) - - def deleteAttr(id: BinaryId) = - delegate.deleteAttr(id) - - def findAttr(id: BinaryId) = - delegate.findAttr(id) + override def findMeta(key: FileKey) = OptionT.none[F, RFileMeta] } def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] = new Impl[F](xa) final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] { - def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] = + def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] = for { now <- Timestamp.current[F] - a <- attrs - fileKey <- makeFileKey(id) fm = RFileMeta( - fileKey, + key, now, - MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream), - ByteSize(a.length), - a.sha256 + MimeType.parse(attrs.contentType.contentType).getOrElse(MimeType.octetStream), + ByteSize(attrs.length), + attrs.sha256 ) _ <- RFileMeta.insert(fm).transact(xa) } yield () - def deleteAttr(id: BinaryId): F[Boolean] = - makeFileKey(id).flatMap(fileKey => - RFileMeta.delete(fileKey).transact(xa).map(_ > 0) - ) + def deleteAttr(key: FileKey): F[Boolean] = + RFileMeta.delete(key).transact(xa).map(_ > 0) - def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] = - findMeta(id).map(fm => - BinaryAttributes( - fm.checksum, - SimpleContentType(fm.mimetype.asString), - fm.length.bytes - ) - ) + def findMeta(key: FileKey): OptionT[F, RFileMeta] = + OptionT(RFileMeta.findById(key).transact(xa)) - def findMeta(id: BinaryId): OptionT[F, RFileMeta] = - OptionT( - makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa)) - ) - - private def makeFileKey(binaryId: BinaryId): F[FileKey] = - Sync[F] - .pure( - BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)) - ) - .rethrow +// private def makeFileKey(binaryId: BinaryId): F[FileKey] = +// Sync[F] +// .pure( +// BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)) +// ) +// .rethrow } } diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index e87d9d3d..15eb4892 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -16,7 +16,7 @@ import docspell.files.TikaMimetype import docspell.logging.Logger import binny._ -import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping} +import binny.fs._ import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping} import scodec.bits.ByteVector @@ -95,7 +95,6 @@ object BinnyUtils { def binaryStore[F[_]: Async]( cfg: FileRepositoryConfig, - attrStore: AttributeStore[F], ds: DataSource, logger: Logger[F] ): BinaryStore[F] = @@ -103,7 +102,7 @@ object BinnyUtils { case FileRepositoryConfig.Database(chunkSize) => val jdbcConfig = JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) - GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore) + GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig) case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) => val keyMapping = S3KeyMapping.constant(bucket) @@ -111,16 +110,16 @@ object BinnyUtils { .default(endpoint, accessKey, secretKey, keyMapping) .copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect) - MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger)) + MinioBinaryStore[F](minioCfg, LoggerAdapter(logger)) case FileRepositoryConfig.Directory(path, chunkSize) => val fsConfig = FsStoreConfig( path, BinnyUtils.TikaContentTypeDetect, - FsStoreConfig.OverwriteMode.Fail, + OverwriteMode.Fail, BinnyUtils.pathMapping, chunkSize ) - FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore) + FsBinaryStore[F](fsConfig, LoggerAdapter(logger)) } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala index 48b30b71..53d96815 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala @@ -13,7 +13,7 @@ import fs2._ import docspell.common._ -import binny.{BinaryAttributeStore, BinaryId, BinaryStore} +import binny.{BinaryId, BinaryStore} import doobie.Transactor trait FileRepository[F[_]] { @@ -45,17 +45,16 @@ object FileRepository { else AttributeStore.empty[F] val log = docspell.logging.getLogger[F] val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId - val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log) - + val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, ds, log) new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun) } def getDelegate[F[_]]( repo: FileRepository[F] - ): Option[(BinaryStore[F], BinaryAttributeStore[F])] = + ): Option[BinaryStore[F]] = repo match { case n: FileRepositoryImpl[F] => - Some((n.bs, n.attrStore)) + Some(n.bs) case _ => None diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala index e605d79c..d612d7ed 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala @@ -30,14 +30,14 @@ final class FileRepositoryImpl[F[_]: Sync]( def findMeta(key: FileKey): F[Option[FileMetadata]] = attrStore - .findMeta(keyFun(key)) + .findMeta(key) .map(rfm => FileMetadata(rfm.id, rfm.created, rfm.mimetype, rfm.length, rfm.checksum) ) .value def delete(key: FileKey): F[Unit] = - bs.delete(keyFun(key)) + bs.delete(keyFun(key)) *> attrStore.deleteAttr(key).void def save( collective: Ident, @@ -48,9 +48,15 @@ final class FileRepositoryImpl[F[_]: Sync]( in => Stream .eval(randomKey(collective, category)) - .flatMap(fkey => - in.through(bs.insertWith(keyFun(fkey), fhint)) ++ Stream.emit(fkey) - ) + .flatMap(fkey => in.through(bs.insertWith(keyFun(fkey))) ++ Stream.emit(fkey)) + .evalTap { key => + val bid = keyFun(key) + bs.computeAttr(bid, fhint) + .run(AttributeName.all) + .semiflatMap(attr => attrStore.saveAttr(key, attr)) + .value + .void + } } def randomKey( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 27d0d9e6..5a03984b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -7,7 +7,7 @@ object Dependencies { val BcryptVersion = "0.4" val BetterMonadicForVersion = "0.3.1" - val BinnyVersion = "0.4.0" + val BinnyVersion = "0.6.0" val CalevVersion = "0.6.3" val CatsVersion = "2.7.0" val CatsEffectVersion = "3.3.12" From f6fd155150d204ae51f68e379703ce6325ec78e2 Mon Sep 17 00:00:00 2001 From: eikek Date: Mon, 27 Jun 2022 13:27:01 +0200 Subject: [PATCH 2/2] Update scribe to 3.9.0 --- .../scala/docspell/logging/impl/JsonWriter.scala | 4 ++-- .../scala/docspell/logging/impl/LogfmtWriter.scala | 4 ++-- .../main/scala/docspell/logging/impl/Record.scala | 13 +++++++++---- .../scala/docspell/logging/impl/ScribeWrapper.scala | 8 ++++---- project/Dependencies.scala | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala index 020fc9f9..653dab0f 100644 --- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala +++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/JsonWriter.scala @@ -13,8 +13,8 @@ import scribe.output.format.OutputFormat import scribe.writer._ final case class JsonWriter(writer: Writer, compact: Boolean = true) extends Writer { - override def write[M]( - record: LogRecord[M], + override def write( + record: LogRecord, output: LogOutput, outputFormat: OutputFormat ): Unit = { diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala index f9f4db12..01ef9adb 100644 --- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala +++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/LogfmtWriter.scala @@ -14,8 +14,8 @@ import scribe.writer._ // https://brandur.org/logfmt final case class LogfmtWriter(writer: Writer) extends Writer { - override def write[M]( - record: LogRecord[M], + override def write( + record: LogRecord, output: LogOutput, outputFormat: OutputFormat ): Unit = { diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala index 68123a5f..165b2333 100644 --- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala +++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/Record.scala @@ -38,18 +38,23 @@ private[impl] case class Record( private[impl] object Record { - def fromLogRecord[M](record: LogRecord[M]): Record = { + def fromLogRecord(record: LogRecord): Record = { val l = record.timeStamp - val traces = record.additionalMessages.collect { + val traces = record.messages.collect { case message: Message[_] if message.value.isInstanceOf[Throwable] => throwable2Trace(message.value.asInstanceOf[Throwable]) } - val additionalMessages = record.additionalMessages.map(_.logOutput.plainText) + val (firstMessage, additionalMessages) = record.messages match { + case h :: rest => + (h.logOutput.plainText, rest.map(_.logOutput.plainText)) + case Nil => + ("", Nil) + } Record( level = record.level.name, levelValue = record.levelValue, - message = record.logOutput.plainText, + message = firstMessage, additionalMessages = additionalMessages, fileName = record.fileName, className = record.className, diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala index d547225c..143fe7fc 100644 --- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala +++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeWrapper.scala @@ -12,7 +12,7 @@ import cats.effect.Sync import docspell.logging.{Level, LogEvent, Logger} import scribe.LoggerSupport -import scribe.message.{LoggableMessage, Message} +import scribe.message.LoggableMessage private[logging] object ScribeWrapper { final class ImplUnsafe(log: scribe.Logger) extends Logger[Id] { @@ -41,10 +41,10 @@ private[logging] object ScribeWrapper { private[this] def convert(ev: LogEvent) = { val level = convertLevel(ev.level) val additional: List[LoggableMessage] = ev.additional.map { - case Right(ex) => Message.static(ex) - case Left(msg) => Message.static(msg) + case Right(ex) => LoggableMessage.throwable2Message(ex) + case Left(msg) => LoggableMessage.string2Message(msg) }.toList - LoggerSupport(level, ev.msg(), additional, ev.pkg, ev.fileName, ev.name, ev.line) + LoggerSupport(level, ev.msg() :: additional, ev.pkg, ev.fileName, ev.name, ev.line) .copy(data = ev.data.toDeferred) } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5a03984b..25b405e3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -43,7 +43,7 @@ object Dependencies { val PureConfigVersion = "0.17.1" val ScalaJavaTimeVersion = "2.4.0" val ScodecBitsVersion = "1.1.31" - val ScribeVersion = "3.8.3" + val ScribeVersion = "3.9.0" val Slf4jVersion = "1.7.36" val SourcecodeVersion = "0.2.8" val StanfordNlpVersion = "4.4.0"