diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 630857fd..c3b4a660 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -65,7 +65,7 @@ object FileCopyTask { if (result.success) s"Successfully copied $allGood files to ${result.counter.size} stores." else - s"Copying files failed for ${failed} files! ${allGood} were copied successfully." + s"Copying files failed for ${failed} files! $allGood were copied successfully." } } @@ -122,8 +122,8 @@ object FileCopyTask { case None => CopyResult.noSourceImpl.pure[F] - case Some((src, srcMeta)) => - to.traverse(FileRepository.getDelegate).map(_.map(_._1)) match { + case Some(src) => + to.traverse(FileRepository.getDelegate) match { case None => CopyResult.noTargetImpl.pure[F] @@ -135,7 +135,7 @@ object FileCopyTask { } def copyTo(to: BinaryStore[F]) = - CopyTool.copyAll[F](log, src, srcMeta, to, 50, maxConcurrent) + CopyTool.copyAll[F](log, src, to, 50, maxConcurrent) logger.info(s"Start copying ${from.config} -> ${to.map(_.config)}") *> targets.traverse(copyTo).map(CopyResult.success) diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index adb21536..dc2b05db 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -60,7 +60,7 @@ object Store { ds.setDriverClassName(jdbc.dbms.driverClass) } xa = HikariTransactor(ds, connectEC) - fr = FileRepository.apply(xa, ds, fileRepoConfig, true) + fr = FileRepository(xa, ds, fileRepoConfig, true) st = new StoreImpl[F](fr, jdbc, schemaCfg, ds, xa) _ <- Resource.eval(st.migrate) } yield st diff --git a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala index 127788b2..54153bd1 100644 --- a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala +++ b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala @@ -18,71 +18,52 @@ import binny._ import doobie._ import doobie.implicits._ -private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] { - def findMeta(id: BinaryId): OptionT[F, RFileMeta] +trait AttributeStore[F[_]] { + def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] + + def deleteAttr(key: FileKey): F[Boolean] + + def findMeta(key: FileKey): OptionT[F, RFileMeta] } private[file] object AttributeStore { def empty[F[_]: Applicative]: AttributeStore[F] = new AttributeStore[F] { - val delegate = BinaryAttributeStore.empty[F] + override def saveAttr(key: FileKey, attrs: BinaryAttributes) = ().pure[F] - def findMeta(id: BinaryId) = - OptionT.none + override def deleteAttr(key: FileKey) = false.pure[F] - def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) = - delegate.saveAttr(id, attrs) - - def deleteAttr(id: BinaryId) = - delegate.deleteAttr(id) - - def findAttr(id: BinaryId) = - delegate.findAttr(id) + override def findMeta(key: FileKey) = OptionT.none[F, RFileMeta] } def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] = new Impl[F](xa) final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] { - def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] = + def saveAttr(key: FileKey, attrs: BinaryAttributes): F[Unit] = for { now <- Timestamp.current[F] - a <- attrs - fileKey <- makeFileKey(id) fm = RFileMeta( - fileKey, + key, now, - MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream), - ByteSize(a.length), - a.sha256 + MimeType.parse(attrs.contentType.contentType).getOrElse(MimeType.octetStream), + ByteSize(attrs.length), + attrs.sha256 ) _ <- RFileMeta.insert(fm).transact(xa) } yield () - def deleteAttr(id: BinaryId): F[Boolean] = - makeFileKey(id).flatMap(fileKey => - RFileMeta.delete(fileKey).transact(xa).map(_ > 0) - ) + def deleteAttr(key: FileKey): F[Boolean] = + RFileMeta.delete(key).transact(xa).map(_ > 0) - def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] = - findMeta(id).map(fm => - BinaryAttributes( - fm.checksum, - SimpleContentType(fm.mimetype.asString), - fm.length.bytes - ) - ) + def findMeta(key: FileKey): OptionT[F, RFileMeta] = + OptionT(RFileMeta.findById(key).transact(xa)) - def findMeta(id: BinaryId): OptionT[F, RFileMeta] = - OptionT( - makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa)) - ) - - private def makeFileKey(binaryId: BinaryId): F[FileKey] = - Sync[F] - .pure( - BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)) - ) - .rethrow +// private def makeFileKey(binaryId: BinaryId): F[FileKey] = +// Sync[F] +// .pure( +// BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)) +// ) +// .rethrow } } diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index e87d9d3d..15eb4892 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -16,7 +16,7 @@ import docspell.files.TikaMimetype import docspell.logging.Logger import binny._ -import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping} +import binny.fs._ import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping} import scodec.bits.ByteVector @@ -95,7 +95,6 @@ object BinnyUtils { def binaryStore[F[_]: Async]( cfg: FileRepositoryConfig, - attrStore: AttributeStore[F], ds: DataSource, logger: Logger[F] ): BinaryStore[F] = @@ -103,7 +102,7 @@ object BinnyUtils { case FileRepositoryConfig.Database(chunkSize) => val jdbcConfig = JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) - GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore) + GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig) case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) => val keyMapping = S3KeyMapping.constant(bucket) @@ -111,16 +110,16 @@ object BinnyUtils { .default(endpoint, accessKey, secretKey, keyMapping) .copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect) - MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger)) + MinioBinaryStore[F](minioCfg, LoggerAdapter(logger)) case FileRepositoryConfig.Directory(path, chunkSize) => val fsConfig = FsStoreConfig( path, BinnyUtils.TikaContentTypeDetect, - FsStoreConfig.OverwriteMode.Fail, + OverwriteMode.Fail, BinnyUtils.pathMapping, chunkSize ) - FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore) + FsBinaryStore[F](fsConfig, LoggerAdapter(logger)) } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala index 48b30b71..53d96815 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala @@ -13,7 +13,7 @@ import fs2._ import docspell.common._ -import binny.{BinaryAttributeStore, BinaryId, BinaryStore} +import binny.{BinaryId, BinaryStore} import doobie.Transactor trait FileRepository[F[_]] { @@ -45,17 +45,16 @@ object FileRepository { else AttributeStore.empty[F] val log = docspell.logging.getLogger[F] val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId - val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log) - + val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, ds, log) new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun) } def getDelegate[F[_]]( repo: FileRepository[F] - ): Option[(BinaryStore[F], BinaryAttributeStore[F])] = + ): Option[BinaryStore[F]] = repo match { case n: FileRepositoryImpl[F] => - Some((n.bs, n.attrStore)) + Some(n.bs) case _ => None diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala index e605d79c..d612d7ed 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala @@ -30,14 +30,14 @@ final class FileRepositoryImpl[F[_]: Sync]( def findMeta(key: FileKey): F[Option[FileMetadata]] = attrStore - .findMeta(keyFun(key)) + .findMeta(key) .map(rfm => FileMetadata(rfm.id, rfm.created, rfm.mimetype, rfm.length, rfm.checksum) ) .value def delete(key: FileKey): F[Unit] = - bs.delete(keyFun(key)) + bs.delete(keyFun(key)) *> attrStore.deleteAttr(key).void def save( collective: Ident, @@ -48,9 +48,15 @@ final class FileRepositoryImpl[F[_]: Sync]( in => Stream .eval(randomKey(collective, category)) - .flatMap(fkey => - in.through(bs.insertWith(keyFun(fkey), fhint)) ++ Stream.emit(fkey) - ) + .flatMap(fkey => in.through(bs.insertWith(keyFun(fkey))) ++ Stream.emit(fkey)) + .evalTap { key => + val bid = keyFun(key) + bs.computeAttr(bid, fhint) + .run(AttributeName.all) + .semiflatMap(attr => attrStore.saveAttr(key, attr)) + .value + .void + } } def randomKey( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 27d0d9e6..5a03984b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -7,7 +7,7 @@ object Dependencies { val BcryptVersion = "0.4" val BetterMonadicForVersion = "0.3.1" - val BinnyVersion = "0.4.0" + val BinnyVersion = "0.6.0" val CalevVersion = "0.6.3" val CatsVersion = "2.7.0" val CatsEffectVersion = "3.3.12"