Store files in different binary stores

modules/joex/src/main/scala/docspell/joex/JoexServer.scala

@@ -16,6 +16,7 @@ import docspell.common.Pools
 import docspell.joex.routes._
 import docspell.pubsub.naive.NaivePubSub
 import docspell.store.Store
+import docspell.store.file.FileRepositoryConfig
 import docspell.store.records.RInternalSetting
 
 import org.http4s.HttpApp
@@ -41,7 +42,7 @@ object JoexServer {
 
       store <- Store.create[F](
         cfg.jdbc,
-        cfg.files.chunkSize,
+        FileRepositoryConfig.Database(cfg.files.chunkSize),
         pools.connectEC
       )
       settings <- Resource.eval(store.transact(RInternalSetting.create))

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -18,6 +18,7 @@ import docspell.restserver.http4s.InternalHeader
 import docspell.restserver.ws.OutputEvent.KeepAlive
 import docspell.restserver.ws.OutputEvent
 import docspell.store.Store
+import docspell.store.file.FileRepositoryConfig
 import docspell.store.records.RInternalSetting
 import org.http4s._
 import org.http4s.blaze.client.BlazeClientBuilder
@@ -73,7 +74,7 @@ object RestServer {
       httpClient <- BlazeClientBuilder[F].resource
       store <- Store.create[F](
         cfg.backend.jdbc,
-        cfg.backend.files.chunkSize,
+        FileRepositoryConfig.Database(cfg.backend.files.chunkSize),
         pools.connectEC
       )
       setting <- Resource.eval(store.transact(RInternalSetting.create))

modules/store/src/main/scala/docspell/store/Store.scala

@@ -12,7 +12,7 @@ import cats.effect._
 import cats.~>
 import fs2._
 
-import docspell.store.file.FileRepository
+import docspell.store.file.{FileRepository, FileRepositoryConfig}
 import docspell.store.impl.StoreImpl
 
 import com.zaxxer.hikari.HikariDataSource
@@ -35,7 +35,7 @@ object Store {
 
   def create[F[_]: Async](
       jdbc: JdbcConfig,
-      chunkSize: Int,
+      fileRepoConfig: FileRepositoryConfig,
       connectEC: ExecutionContext
   ): Resource[F, Store[F]] = {
     val acquire = Sync[F].delay(new HikariDataSource())
@@ -50,7 +50,7 @@ object Store {
         ds.setDriverClassName(jdbc.driverClass)
       }
       xa = HikariTransactor(ds, connectEC)
-      fr = FileRepository.genericJDBC(xa, ds, chunkSize)
+      fr = FileRepository.apply(xa, ds, fileRepoConfig)
       st = new StoreImpl[F](fr, jdbc, xa)
       _ <- Resource.eval(st.migrate)
     } yield st

modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala

@@ -6,12 +6,19 @@
 
 package docspell.store.file
 
-import docspell.common
+import javax.sql.DataSource
+
+import cats.effect._
+import fs2.io.file.Path
+
 import docspell.common._
 import docspell.files.TikaMimetype
 import docspell.logging.Logger
 
 import binny._
+import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping}
+import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
+import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping}
 import scodec.bits.ByteVector
 
 private[store] object BinnyUtils {
@@ -26,7 +33,7 @@ private[store] object BinnyUtils {
           coll <- Ident.fromString(cId)
           cat <- FileCategory.fromString(catId)
           file <- Ident.fromString(fId)
-        } yield common.FileKey(coll, cat, file)
+        } yield FileKey(coll, cat, file)
       case _ =>
         Left(s"Invalid format for file-key: $bid")
     }
@@ -57,4 +64,55 @@ private[store] object BinnyUtils {
           .asString
       )
   }
+
+  val pathMapping: PathMapping = {
+    import binny.fs.PathMapping.syntax._
+
+    def toPath(base: Path, binaryId: BinaryId): Path = {
+      val fkey = unsafeBinaryIdToFileKey(binaryId)
+      base / fkey.collective.id / fkey.category.id.id / fkey.id.id / "file"
+    }
+
+    def toId(file: Path): Option[BinaryId] =
+      for {
+        id <- file.parent
+        cat <- id.parent
+        fcat <- FileCategory.fromString(cat.asId.id).toOption
+        coll <- cat.parent
+        fkey = FileKey(Ident.unsafe(coll.asId.id), fcat, Ident.unsafe(id.asId.id))
+      } yield fileKeyToBinaryId(fkey)
+
+    PathMapping(toPath)(toId)
+  }
+
+  def binaryStore[F[_]: Async](
+      cfg: FileRepositoryConfig,
+      attrStore: AttributeStore[F],
+      ds: DataSource,
+      logger: Logger[F]
+  ): BinaryStore[F] =
+    cfg match {
+      case FileRepositoryConfig.Database(chunkSize) =>
+        val jdbcConfig =
+          JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect)
+        GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore)
+
+      case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) =>
+        val keyMapping = S3KeyMapping.constant(bucket)
+        val minioCfg = MinioConfig
+          .default(endpoint, accessKey, secretKey, keyMapping)
+          .copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect)
+
+        MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger))
+
+      case FileRepositoryConfig.Directory(path, chunkSize) =>
+        val fsConfig = FsStoreConfig(
+          path,
+          BinnyUtils.TikaContentTypeDetect,
+          FsStoreConfig.OverwriteMode.Fail,
+          BinnyUtils.pathMapping,
+          chunkSize
+        )
+        FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore)
+    }
 }

modules/store/src/main/scala/docspell/store/file/FileRepository.scala

@@ -13,8 +13,7 @@ import fs2._
 
 import docspell.common._
 
-import binny.BinaryId
-import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
+import binny.{BinaryId, BinaryStore}
 import doobie.Transactor
 
 trait FileRepository[F[_]] {
@@ -33,16 +32,15 @@ trait FileRepository[F[_]] {
 
 object FileRepository {
 
-  def genericJDBC[F[_]: Sync](
+  def apply[F[_]: Async](
       xa: Transactor[F],
       ds: DataSource,
-      chunkSize: Int
+      cfg: FileRepositoryConfig
   ): FileRepository[F] = {
     val attrStore = new AttributeStore[F](xa)
-    val cfg = JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect)
     val log = docspell.logging.getLogger[F]
-    val binStore = GenericJdbcStore[F](ds, BinnyUtils.LoggerAdapter(log), cfg, attrStore)
     val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId
+    val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log)
 
     new FileRepositoryImpl[F](binStore, attrStore, keyFun)
   }

modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.store.file
+
+import fs2.io.file.Path
+
+sealed trait FileRepositoryConfig {}
+
+object FileRepositoryConfig {
+
+  final case class Database(chunkSize: Int) extends FileRepositoryConfig
+
+  final case class S3(
+      endpoint: String,
+      accessKey: String,
+      secretKey: String,
+      bucketName: String,
+      chunkSize: Int
+  ) extends FileRepositoryConfig
+
+  final case class Directory(path: Path, chunkSize: Int) extends FileRepositoryConfig
+
+}

modules/store/src/test/scala/docspell/store/StoreFixture.scala

@@ -11,7 +11,7 @@ import javax.sql.DataSource
 import cats.effect._
 
 import docspell.common.LenientUri
-import docspell.store.file.FileRepository
+import docspell.store.file.{FileRepository, FileRepositoryConfig}
 import docspell.store.impl.StoreImpl
 import docspell.store.migrate.FlywayMigrate
 
@@ -67,7 +67,8 @@ object StoreFixture {
     for {
       ds <- dataSource(jdbc)
       xa <- makeXA(ds)
-      fr = FileRepository.genericJDBC[IO](xa, ds, 64 * 1024)
+      cfg = FileRepositoryConfig.Database(64 * 1024)
+      fr = FileRepository[IO](xa, ds, cfg)
       store = new StoreImpl[IO](fr, jdbc, xa)
       _ <- Resource.eval(store.migrate)
     } yield store

project/Dependencies.scala

@@ -307,7 +307,8 @@ object Dependencies {
   val binny = Seq(
     "com.github.eikek" %% "binny-core" % BinnyVersion,
     "com.github.eikek" %% "binny-jdbc" % BinnyVersion,
-    "com.github.eikek" %% "binny-minio" % BinnyVersion
+    "com.github.eikek" %% "binny-minio" % BinnyVersion,
+    "com.github.eikek" %% "binny-fs" % BinnyVersion
   )
 
   // https://github.com/flyway/flyway