TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add task to copy files

Browse Source
This commit is contained in:
eikek
2022-03-08 00:11:40 +01:00
parent e82b00c582
commit 422c0905dc
28 changed files with 512 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
modules/store/src/main/scala/docspell/store/Store.scala
View File

@ -28,6 +28,11 @@ trait Store[F[_]] {
def fileRepo: FileRepository[F]
def createFileRepository(
cfg: FileRepositoryConfig,
withAttributeStore: Boolean
): FileRepository[F]
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult]
}
@ -50,8 +55,8 @@ object Store {
ds.setDriverClassName(jdbc.driverClass)
}
xa = HikariTransactor(ds, connectEC)
fr = FileRepository.apply(xa, ds, fileRepoConfig)
st = new StoreImpl[F](fr, jdbc, xa)
fr = FileRepository.apply(xa, ds, fileRepoConfig, true)
st = new StoreImpl[F](fr, jdbc, ds, xa)
_ <- Resource.eval(st.migrate)
} yield st
}

104
modules/store/src/main/scala/docspell/store/file/AttributeStore.scala
View File

@ -6,6 +6,7 @@
package docspell.store.file
import cats.Applicative
import cats.data.OptionT
import cats.effect._
import cats.implicits._
@ -17,40 +18,71 @@ import binny._
import doobie._
import doobie.implicits._
final private[file] class AttributeStore[F[_]: Sync](xa: Transactor[F])
extends BinaryAttributeStore[F] {
def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] =
for {
now <- Timestamp.current[F]
a <- attrs
fileKey <- makeFileKey(id)
fm = RFileMeta(
fileKey,
now,
MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream),
ByteSize(a.length),
a.sha256
)
_ <- RFileMeta.insert(fm).transact(xa)
} yield ()
def deleteAttr(id: BinaryId): F[Boolean] =
makeFileKey(id).flatMap(fileKey => RFileMeta.delete(fileKey).transact(xa).map(_ > 0))
def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] =
findMeta(id).map(fm =>
BinaryAttributes(
fm.checksum,
SimpleContentType(fm.mimetype.asString),
fm.length.bytes
)
)
def findMeta(id: BinaryId): OptionT[F, RFileMeta] =
OptionT(makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa)))
private def makeFileKey(binaryId: BinaryId): F[FileKey] =
Sync[F]
.pure(BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)))
.rethrow
private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] {
def findMeta(id: BinaryId): OptionT[F, RFileMeta]
}
private[file] object AttributeStore {
def empty[F[_]: Applicative]: AttributeStore[F] =
new AttributeStore[F] {
val delegate = BinaryAttributeStore.empty[F]
def findMeta(id: BinaryId) =
OptionT.none
def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) =
delegate.saveAttr(id, attrs)
def deleteAttr(id: BinaryId) =
delegate.deleteAttr(id)
def findAttr(id: BinaryId) =
delegate.findAttr(id)
}
def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] =
new Impl[F](xa)
final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] {
def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] =
for {
now <- Timestamp.current[F]
a <- attrs
fileKey <- makeFileKey(id)
fm = RFileMeta(
fileKey,
now,
MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream),
ByteSize(a.length),
a.sha256
)
_ <- RFileMeta.insert(fm).transact(xa)
} yield ()
def deleteAttr(id: BinaryId): F[Boolean] =
makeFileKey(id).flatMap(fileKey =>
RFileMeta.delete(fileKey).transact(xa).map(_ > 0)
)
def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] =
findMeta(id).map(fm =>
BinaryAttributes(
fm.checksum,
SimpleContentType(fm.mimetype.asString),
fm.length.bytes
)
)
def findMeta(id: BinaryId): OptionT[F, RFileMeta] =
OptionT(
makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa))
)
private def makeFileKey(binaryId: BinaryId): F[FileKey] =
Sync[F]
.pure(
BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))
)
.rethrow
}
}

2
modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala
View File

@ -21,7 +21,7 @@ import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping}
import scodec.bits.ByteVector
private[store] object BinnyUtils {
object BinnyUtils {
def fileKeyToBinaryId(fk: FileKey): BinaryId =
BinaryId(s"${fk.collective.id}/${fk.category.id.id}/${fk.id.id}")

24
modules/store/src/main/scala/docspell/store/file/FileRepository.scala
View File

@ -13,10 +13,12 @@ import fs2._
import docspell.common._
import binny.{BinaryId, BinaryStore}
import binny.{BinaryAttributeStore, BinaryId, BinaryStore}
import doobie.Transactor
trait FileRepository[F[_]] {
def config: FileRepositoryConfig
def getBytes(key: FileKey): Stream[F, Byte]
def findMeta(key: FileKey): F[Option[FileMetadata]]
@ -35,13 +37,27 @@ object FileRepository {
def apply[F[_]: Async](
xa: Transactor[F],
ds: DataSource,
cfg: FileRepositoryConfig
cfg: FileRepositoryConfig,
withAttributeStore: Boolean
): FileRepository[F] = {
val attrStore = new AttributeStore[F](xa)
val attrStore =
if (withAttributeStore) AttributeStore[F](xa)
else AttributeStore.empty[F]
val log = docspell.logging.getLogger[F]
val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId
val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log)
new FileRepositoryImpl[F](binStore, attrStore, keyFun)
new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun)
}
def getDelegate[F[_]](
repo: FileRepository[F]
): Option[(BinaryStore[F], BinaryAttributeStore[F])] =
repo match {
case n: FileRepositoryImpl[F] =>
Some((n.bs, n.attrStore))
case _ =>
None
}
}

11
modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala
View File

@ -8,6 +8,8 @@ package docspell.store.file
import fs2.io.file.Path
import docspell.common.FileStoreConfig
sealed trait FileRepositoryConfig {}
object FileRepositoryConfig {
@ -24,4 +26,13 @@ object FileRepositoryConfig {
final case class Directory(path: Path, chunkSize: Int) extends FileRepositoryConfig
def fromFileStoreConfig(chunkSize: Int, cfg: FileStoreConfig): FileRepositoryConfig =
cfg match {
case FileStoreConfig.DefaultDatabase(_) =>
FileRepositoryConfig.Database(chunkSize)
case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) =>
FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize)
case FileStoreConfig.FileSystem(_, directory) =>
FileRepositoryConfig.Directory(directory, chunkSize)
}
}

5
modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala
View File

@ -16,8 +16,9 @@ import docspell.common._
import binny._
final class FileRepositoryImpl[F[_]: Sync](
bs: BinaryStore[F],
attrStore: AttributeStore[F],
val config: FileRepositoryConfig,
val bs: BinaryStore[F],
val attrStore: AttributeStore[F],
keyFun: FileKey => BinaryId
) extends FileRepository[F] {

11
modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala
View File

@ -6,12 +6,14 @@
package docspell.store.impl
import javax.sql.DataSource
import cats.arrow.FunctionK
import cats.effect.Async
import cats.implicits._
import cats.~>
import docspell.store.file.FileRepository
import docspell.store.file.{FileRepository, FileRepositoryConfig}
import docspell.store.migrate.FlywayMigrate
import docspell.store.{AddResult, JdbcConfig, Store}
@ -21,9 +23,16 @@ import doobie.implicits._
final class StoreImpl[F[_]: Async](
val fileRepo: FileRepository[F],
jdbc: JdbcConfig,
ds: DataSource,
xa: Transactor[F]
) extends Store[F] {
def createFileRepository(
cfg: FileRepositoryConfig,
withAttributeStore: Boolean
): FileRepository[F] =
FileRepository(xa, ds, cfg, withAttributeStore)
def transform: ConnectionIO ~> F =
FunctionK.lift(transact)

2
modules/store/src/test/scala/docspell/store/StoreFixture.scala
View File

@ -69,7 +69,7 @@ object StoreFixture {
xa <- makeXA(ds)
cfg = FileRepositoryConfig.Database(64 * 1024)
fr = FileRepository[IO](xa, ds, cfg)
store = new StoreImpl[IO](fr, jdbc, xa)
store = new StoreImpl[IO](fr, jdbc, ds, xa)
_ <- Resource.eval(store.migrate)
} yield store
}