diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index ef06f730..1d0458d4 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -16,6 +16,7 @@ import docspell.common.Pools import docspell.joex.routes._ import docspell.pubsub.naive.NaivePubSub import docspell.store.Store +import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s.HttpApp @@ -41,7 +42,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, - cfg.files.chunkSize, + FileRepositoryConfig.Database(cfg.files.chunkSize), pools.connectEC ) settings <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 0b58a584..57c77707 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -18,6 +18,7 @@ import docspell.restserver.http4s.InternalHeader import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.restserver.ws.OutputEvent import docspell.store.Store +import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder @@ -73,7 +74,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, - cfg.backend.files.chunkSize, + FileRepositoryConfig.Database(cfg.backend.files.chunkSize), pools.connectEC ) setting <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index 3e54a7a2..24c80b98 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -12,7 +12,7 @@ import cats.effect._ import cats.~> import fs2._ -import docspell.store.file.FileRepository +import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.impl.StoreImpl import com.zaxxer.hikari.HikariDataSource @@ -35,7 +35,7 @@ object Store { def create[F[_]: Async]( jdbc: JdbcConfig, - chunkSize: Int, + fileRepoConfig: FileRepositoryConfig, connectEC: ExecutionContext ): Resource[F, Store[F]] = { val acquire = Sync[F].delay(new HikariDataSource()) @@ -50,7 +50,7 @@ object Store { ds.setDriverClassName(jdbc.driverClass) } xa = HikariTransactor(ds, connectEC) - fr = FileRepository.genericJDBC(xa, ds, chunkSize) + fr = FileRepository.apply(xa, ds, fileRepoConfig) st = new StoreImpl[F](fr, jdbc, xa) _ <- Resource.eval(st.migrate) } yield st diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index eef07da3..bdd09f48 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -6,12 +6,19 @@ package docspell.store.file -import docspell.common +import javax.sql.DataSource + +import cats.effect._ +import fs2.io.file.Path + import docspell.common._ import docspell.files.TikaMimetype import docspell.logging.Logger import binny._ +import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping} +import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping} import scodec.bits.ByteVector private[store] object BinnyUtils { @@ -26,7 +33,7 @@ private[store] object BinnyUtils { coll <- Ident.fromString(cId) cat <- FileCategory.fromString(catId) file <- Ident.fromString(fId) - } yield common.FileKey(coll, cat, file) + } yield FileKey(coll, cat, file) case _ => Left(s"Invalid format for file-key: $bid") } @@ -57,4 +64,55 @@ private[store] object BinnyUtils { .asString ) } + + val pathMapping: PathMapping = { + import binny.fs.PathMapping.syntax._ + + def toPath(base: Path, binaryId: BinaryId): Path = { + val fkey = unsafeBinaryIdToFileKey(binaryId) + base / fkey.collective.id / fkey.category.id.id / fkey.id.id / "file" + } + + def toId(file: Path): Option[BinaryId] = + for { + id <- file.parent + cat <- id.parent + fcat <- FileCategory.fromString(cat.asId.id).toOption + coll <- cat.parent + fkey = FileKey(Ident.unsafe(coll.asId.id), fcat, Ident.unsafe(id.asId.id)) + } yield fileKeyToBinaryId(fkey) + + PathMapping(toPath)(toId) + } + + def binaryStore[F[_]: Async]( + cfg: FileRepositoryConfig, + attrStore: AttributeStore[F], + ds: DataSource, + logger: Logger[F] + ): BinaryStore[F] = + cfg match { + case FileRepositoryConfig.Database(chunkSize) => + val jdbcConfig = + JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) + GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore) + + case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) => + val keyMapping = S3KeyMapping.constant(bucket) + val minioCfg = MinioConfig + .default(endpoint, accessKey, secretKey, keyMapping) + .copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect) + + MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger)) + + case FileRepositoryConfig.Directory(path, chunkSize) => + val fsConfig = FsStoreConfig( + path, + BinnyUtils.TikaContentTypeDetect, + FsStoreConfig.OverwriteMode.Fail, + BinnyUtils.pathMapping, + chunkSize + ) + FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore) + } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala index b3da6da3..b8bef362 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala @@ -13,8 +13,7 @@ import fs2._ import docspell.common._ -import binny.BinaryId -import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.{BinaryId, BinaryStore} import doobie.Transactor trait FileRepository[F[_]] { @@ -33,16 +32,15 @@ trait FileRepository[F[_]] { object FileRepository { - def genericJDBC[F[_]: Sync]( + def apply[F[_]: Async]( xa: Transactor[F], ds: DataSource, - chunkSize: Int + cfg: FileRepositoryConfig ): FileRepository[F] = { val attrStore = new AttributeStore[F](xa) - val cfg = JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) val log = docspell.logging.getLogger[F] - val binStore = GenericJdbcStore[F](ds, BinnyUtils.LoggerAdapter(log), cfg, attrStore) val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId + val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log) new FileRepositoryImpl[F](binStore, attrStore, keyFun) } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala new file mode 100644 index 00000000..5c575e3d --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.file + +import fs2.io.file.Path + +sealed trait FileRepositoryConfig {} + +object FileRepositoryConfig { + + final case class Database(chunkSize: Int) extends FileRepositoryConfig + + final case class S3( + endpoint: String, + accessKey: String, + secretKey: String, + bucketName: String, + chunkSize: Int + ) extends FileRepositoryConfig + + final case class Directory(path: Path, chunkSize: Int) extends FileRepositoryConfig + +} diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala index 91441701..dc11ff41 100644 --- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -11,7 +11,7 @@ import javax.sql.DataSource import cats.effect._ import docspell.common.LenientUri -import docspell.store.file.FileRepository +import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.impl.StoreImpl import docspell.store.migrate.FlywayMigrate @@ -67,7 +67,8 @@ object StoreFixture { for { ds <- dataSource(jdbc) xa <- makeXA(ds) - fr = FileRepository.genericJDBC[IO](xa, ds, 64 * 1024) + cfg = FileRepositoryConfig.Database(64 * 1024) + fr = FileRepository[IO](xa, ds, cfg) store = new StoreImpl[IO](fr, jdbc, xa) _ <- Resource.eval(store.migrate) } yield store diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 34d7f735..a9093d36 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -307,7 +307,8 @@ object Dependencies { val binny = Seq( "com.github.eikek" %% "binny-core" % BinnyVersion, "com.github.eikek" %% "binny-jdbc" % BinnyVersion, - "com.github.eikek" %% "binny-minio" % BinnyVersion + "com.github.eikek" %% "binny-minio" % BinnyVersion, + "com.github.eikek" %% "binny-fs" % BinnyVersion ) // https://github.com/flyway/flyway