Extract OItemSearch from OItem

This commit is contained in:
Eike Kettner 2020-06-15 23:13:14 +02:00
parent e429f9554d
commit 7a3d2e4dc6
8 changed files with 250 additions and 221 deletions

View File

@ -24,6 +24,7 @@ trait BackendApp[F[_]] {
def node: ONode[F]
def job: OJob[F]
def item: OItem[F]
def itemSearch: OItemSearch[F]
def mail: OMail[F]
def joex: OJoex[F]
def userTask: OUserTask[F]
@ -38,20 +39,21 @@ object BackendApp {
blocker: Blocker
): Resource[F, BackendApp[F]] =
for {
utStore <- UserTaskStore(store)
queue <- JobQueue(store)
loginImpl <- Login[F](store)
signupImpl <- OSignup[F](store)
collImpl <- OCollective[F](store)
sourceImpl <- OSource[F](store)
tagImpl <- OTag[F](store)
equipImpl <- OEquipment[F](store)
orgImpl <- OOrganization(store)
joexImpl <- OJoex.create(httpClientEc, store)
uploadImpl <- OUpload(store, queue, cfg.files, joexImpl)
nodeImpl <- ONode(store)
jobImpl <- OJob(store, joexImpl)
itemImpl <- OItem(store)
utStore <- UserTaskStore(store)
queue <- JobQueue(store)
loginImpl <- Login[F](store)
signupImpl <- OSignup[F](store)
collImpl <- OCollective[F](store)
sourceImpl <- OSource[F](store)
tagImpl <- OTag[F](store)
equipImpl <- OEquipment[F](store)
orgImpl <- OOrganization(store)
joexImpl <- OJoex.create(httpClientEc, store)
uploadImpl <- OUpload(store, queue, cfg.files, joexImpl)
nodeImpl <- ONode(store)
jobImpl <- OJob(store, joexImpl)
itemImpl <- OItem(store)
itemSearchImpl <- OItemSearch(store)
javaEmil =
JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))
mailImpl <- OMail(store, javaEmil)
@ -68,6 +70,7 @@ object BackendApp {
val node = nodeImpl
val job = jobImpl
val item = itemImpl
val itemSearch = itemSearchImpl
val mail = mailImpl
val joex = joexImpl
val userTask = userTaskImpl

View File

@ -1,6 +1,5 @@
package docspell.backend.ops
import fs2.Stream
import cats.data.OptionT
import cats.implicits._
import cats.effect.{Effect, Resource}
@ -8,41 +7,11 @@ import doobie._
import doobie.implicits._
import docspell.store.{AddResult, Store}
import docspell.store.queries.{QAttachment, QItem}
import OItem.{
AttachmentArchiveData,
AttachmentData,
AttachmentSourceData,
Batch,
ItemData,
ListItem,
ListItemWithTags,
Query
}
import bitpeace.{FileMeta, RangeDef}
import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp}
import docspell.store.records._
trait OItem[F[_]] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
def findAttachmentSource(
id: Ident,
collective: Ident
): F[Option[AttachmentSourceData[F]]]
def findAttachmentArchive(
id: Ident,
collective: Ident
): F[Option[AttachmentArchiveData[F]]]
/** Sets the given tags (removing all existing ones). */
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult]
@ -85,12 +54,6 @@ trait OItem[F[_]] {
def deleteItem(itemId: Ident, collective: Ident): F[Int]
def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]]
def findByFileCollective(checksum: String, collective: Ident): F[Vector[RItem]]
def findByFileSource(checksum: String, sourceId: Ident): F[Vector[RItem]]
def deleteAttachment(id: Ident, collective: Ident): F[Int]
def moveAttachmentBefore(itemId: Ident, source: Ident, target: Ident): F[AddResult]
@ -104,51 +67,6 @@ trait OItem[F[_]] {
object OItem {
type Query = QItem.Query
val Query = QItem.Query
type Batch = QItem.Batch
val Batch = QItem.Batch
type ListItem = QItem.ListItem
val ListItem = QItem.ListItem
type ListItemWithTags = QItem.ListItemWithTags
val ListItemWithTags = QItem.ListItemWithTags
type ItemData = QItem.ItemData
val ItemData = QItem.ItemData
trait BinaryData[F[_]] {
def data: Stream[F, Byte]
def name: Option[String]
def meta: FileMeta
def fileId: Ident
}
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
extends BinaryData[F] {
val name = ra.name
val fileId = ra.fileId
}
case class AttachmentSourceData[F[_]](
rs: RAttachmentSource,
meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId
}
case class AttachmentArchiveData[F[_]](
rs: RAttachmentArchive,
meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId
}
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] =
for {
otag <- OTag(store)
@ -165,90 +83,6 @@ object OItem {
.attempt
.map(AddResult.fromUpdate)
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store
.transact(QItem.findItem(id))
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, batch).take(batch.limit.toLong))
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
store
.transact(QItem.findItemsWithTags(q, batch).take(batch.limit.toLong))
.compile
.toVector
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentData[F]]).pure[F]
})
def findAttachmentSource(
id: Ident,
collective: Ident
): F[Option[AttachmentSourceData[F]]] =
store
.transact(RAttachmentSource.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentSourceData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentSourceData[F]]).pure[F]
})
def findAttachmentArchive(
id: Ident,
collective: Ident
): F[Option[AttachmentArchiveData[F]]] =
store
.transact(RAttachmentArchive.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentArchiveData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentArchiveData[F]]).pure[F]
})
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
store.bitpeace
.get(fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m => f(m))
)
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for {
cid <- RItem.getCollective(item)
@ -464,18 +298,6 @@ object OItem {
def getProposals(item: Ident, collective: Ident): F[MetaProposalList] =
store.transact(QAttachment.getMetaProposals(item, collective))
def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]] =
store.transact(QAttachment.getAttachmentMeta(id, collective))
def findByFileCollective(checksum: String, collective: Ident): F[Vector[RItem]] =
store.transact(QItem.findByChecksum(checksum, collective))
def findByFileSource(checksum: String, sourceId: Ident): F[Vector[RItem]] =
store.transact((for {
coll <- OptionT(RSource.findCollective(sourceId))
items <- OptionT.liftF(QItem.findByChecksum(checksum, coll))
} yield items).getOrElse(Vector.empty))
def deleteAttachment(id: Ident, collective: Ident): F[Int] =
QAttachment.deleteSingleAttachment(store)(id, collective)

View File

@ -0,0 +1,199 @@
package docspell.backend.ops
import fs2.Stream
import cats.data.OptionT
import cats.implicits._
import cats.effect.{Effect, Resource}
import doobie.implicits._
import docspell.store.Store
import docspell.store.queries.{QAttachment, QItem}
import OItemSearch.{
AttachmentArchiveData,
AttachmentData,
AttachmentSourceData,
Batch,
ItemData,
ListItem,
ListItemWithTags,
Query
}
import bitpeace.{FileMeta, RangeDef}
import docspell.common._
import docspell.store.records._
trait OItemSearch[F[_]] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]]
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
def findAttachmentSource(
id: Ident,
collective: Ident
): F[Option[AttachmentSourceData[F]]]
def findAttachmentArchive(
id: Ident,
collective: Ident
): F[Option[AttachmentArchiveData[F]]]
def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]]
def findByFileCollective(checksum: String, collective: Ident): F[Vector[RItem]]
def findByFileSource(checksum: String, sourceId: Ident): F[Vector[RItem]]
}
object OItemSearch {
type Query = QItem.Query
val Query = QItem.Query
type Batch = QItem.Batch
val Batch = QItem.Batch
type ListItem = QItem.ListItem
val ListItem = QItem.ListItem
type ListItemWithTags = QItem.ListItemWithTags
val ListItemWithTags = QItem.ListItemWithTags
type ItemData = QItem.ItemData
val ItemData = QItem.ItemData
trait BinaryData[F[_]] {
def data: Stream[F, Byte]
def name: Option[String]
def meta: FileMeta
def fileId: Ident
}
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
extends BinaryData[F] {
val name = ra.name
val fileId = ra.fileId
}
case class AttachmentSourceData[F[_]](
rs: RAttachmentSource,
meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId
}
case class AttachmentArchiveData[F[_]](
rs: RAttachmentArchive,
meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId
}
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItemSearch[F]] =
Resource.pure[F, OItemSearch[F]](new OItemSearch[F] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store
.transact(QItem.findItem(id))
.map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, batch: Batch): F[Vector[ListItem]] =
store
.transact(QItem.findItems(q, batch).take(batch.limit.toLong))
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
store
.transact(QItem.findItemsWithTags(q, batch).take(batch.limit.toLong))
.compile
.toVector
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentData[F]]).pure[F]
})
def findAttachmentSource(
id: Ident,
collective: Ident
): F[Option[AttachmentSourceData[F]]] =
store
.transact(RAttachmentSource.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentSourceData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentSourceData[F]]).pure[F]
})
def findAttachmentArchive(
id: Ident,
collective: Ident
): F[Option[AttachmentArchiveData[F]]] =
store
.transact(RAttachmentArchive.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
makeBinaryData(ra.fileId) { m =>
AttachmentArchiveData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
}
case None =>
(None: Option[AttachmentArchiveData[F]]).pure[F]
})
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
store.bitpeace
.get(fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m => f(m))
)
def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]] =
store.transact(QAttachment.getAttachmentMeta(id, collective))
def findByFileCollective(checksum: String, collective: Ident): F[Vector[RItem]] =
store.transact(QItem.findByChecksum(checksum, collective))
def findByFileSource(checksum: String, sourceId: Ident): F[Vector[RItem]] =
store.transact((for {
coll <- OptionT(RSource.findCollective(sourceId))
items <- OptionT.liftF(QItem.findByChecksum(checksum, coll))
} yield items).getOrElse(Vector.empty))
})
}

View File

@ -8,7 +8,7 @@ import emil.markdown._
import emil.javamail.syntax._
import docspell.common._
import docspell.backend.ops.OItem.Batch
import docspell.backend.ops.OItemSearch.Batch
import docspell.store.records._
import docspell.store.queries.QItem
import docspell.joex.scheduler.{Context, Task}

View File

@ -14,7 +14,7 @@ import bitpeace.FileMeta
import docspell.backend.ops.OCollective.{InsightData, PassChangeResult}
import docspell.backend.ops.OJob.JobCancelResult
import docspell.backend.ops.OUpload.{UploadData, UploadMeta, UploadResult}
import docspell.backend.ops.{OItem, OJob, OOrganization, OUpload}
import docspell.backend.ops.{OItemSearch, OJob, OOrganization, OUpload}
import docspell.store.AddResult
import org.http4s.multipart.Multipart
import org.http4s.headers.`Content-Type`
@ -67,7 +67,7 @@ trait Conversions {
}
// item detail
def mkItemDetail(data: OItem.ItemData): ItemDetail =
def mkItemDetail(data: OItemSearch.ItemData): ItemDetail =
ItemDetail(
data.item.id,
data.item.direction,
@ -90,7 +90,9 @@ trait Conversions {
data.tags.map(mkTag).toList
)
def mkAttachment(item: OItem.ItemData)(ra: RAttachment, m: FileMeta): Attachment = {
def mkAttachment(
item: OItemSearch.ItemData
)(ra: RAttachment, m: FileMeta): Attachment = {
val converted =
item.sources.find(_._1.id == ra.id).exists(_._2.checksum != m.checksum)
Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString), converted)
@ -104,8 +106,8 @@ trait Conversions {
// item list
def mkQuery(m: ItemSearch, coll: Ident): OItem.Query =
OItem.Query(
def mkQuery(m: ItemSearch, coll: Ident): OItemSearch.Query =
OItemSearch.Query(
coll,
m.name,
if (m.inbox) Seq(ItemState.Created)
@ -125,10 +127,10 @@ trait Conversions {
None
)
def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = {
def mkItemList(v: Vector[OItemSearch.ListItem]): ItemLightList = {
val groups = v.groupBy(item => item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItem.ListItem])): ItemLightGroup =
def mkGroup(g: (String, Vector[OItemSearch.ListItem])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLight).toList)
val gs =
@ -136,10 +138,10 @@ trait Conversions {
ItemLightList(gs)
}
def mkItemListWithTags(v: Vector[OItem.ListItemWithTags]): ItemLightList = {
def mkItemListWithTags(v: Vector[OItemSearch.ListItemWithTags]): ItemLightList = {
val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItem.ListItemWithTags])): ItemLightGroup =
def mkGroup(g: (String, Vector[OItemSearch.ListItemWithTags])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList)
val gs =
@ -147,7 +149,7 @@ trait Conversions {
ItemLightList(gs)
}
def mkItemLight(i: OItem.ListItem): ItemLight =
def mkItemLight(i: OItemSearch.ListItem): ItemLight =
ItemLight(
i.id,
i.name,
@ -164,7 +166,7 @@ trait Conversions {
Nil
)
def mkItemLightWithTags(i: OItem.ListItemWithTags): ItemLight =
def mkItemLightWithTags(i: OItemSearch.ListItemWithTags): ItemLight =
mkItemLight(i.item).copy(tags = i.tags.map(mkTag))
// job

View File

@ -12,7 +12,7 @@ import org.http4s.circe.CirceEntityEncoder._
import org.http4s.circe.CirceEntityDecoder._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.backend.ops.OItem
import docspell.backend.ops._
import docspell.common.Ident
import docspell.restapi.model._
import docspell.restserver.conv.Conversions
@ -26,7 +26,7 @@ object AttachmentRoutes {
def withResponseHeaders(
resp: F[Response[F]]
)(data: OItem.BinaryData[F]): F[Response[F]] = {
)(data: OItemSearch.BinaryData[F]): F[Response[F]] = {
val mt = MediaType.unsafeParse(data.meta.mimetype.asString)
val ctype = `Content-Type`(mt)
val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length)
@ -40,13 +40,13 @@ object AttachmentRoutes {
)
}
def makeByteResp(data: OItem.BinaryData[F]): F[Response[F]] =
def makeByteResp(data: OItemSearch.BinaryData[F]): F[Response[F]] =
withResponseHeaders(Ok(data.data.take(data.meta.length)))(data)
HttpRoutes.of {
case HEAD -> Root / Ident(id) =>
for {
fileData <- backend.item.findAttachment(id, user.account.collective)
fileData <- backend.itemSearch.findAttachment(id, user.account.collective)
resp <-
fileData
.map(data => withResponseHeaders(Ok())(data))
@ -55,7 +55,7 @@ object AttachmentRoutes {
case req @ GET -> Root / Ident(id) =>
for {
fileData <- backend.item.findAttachment(id, user.account.collective)
fileData <- backend.itemSearch.findAttachment(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
resp <-
@ -69,7 +69,7 @@ object AttachmentRoutes {
case HEAD -> Root / Ident(id) / "original" =>
for {
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
fileData <- backend.itemSearch.findAttachmentSource(id, user.account.collective)
resp <-
fileData
.map(data => withResponseHeaders(Ok())(data))
@ -78,7 +78,7 @@ object AttachmentRoutes {
case req @ GET -> Root / Ident(id) / "original" =>
for {
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
fileData <- backend.itemSearch.findAttachmentSource(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
resp <-
@ -92,7 +92,8 @@ object AttachmentRoutes {
case HEAD -> Root / Ident(id) / "archive" =>
for {
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
fileData <-
backend.itemSearch.findAttachmentArchive(id, user.account.collective)
resp <-
fileData
.map(data => withResponseHeaders(Ok())(data))
@ -101,7 +102,8 @@ object AttachmentRoutes {
case req @ GET -> Root / Ident(id) / "archive" =>
for {
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
fileData <-
backend.itemSearch.findAttachmentArchive(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
resp <-
@ -122,7 +124,7 @@ object AttachmentRoutes {
case GET -> Root / Ident(id) / "meta" =>
for {
rm <- backend.item.findAttachmentMeta(id, user.account.collective)
rm <- backend.itemSearch.findAttachmentMeta(id, user.account.collective)
md = rm.map(Conversions.mkAttachmentMeta)
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp

View File

@ -21,8 +21,9 @@ object CheckFileRoutes {
HttpRoutes.of {
case GET -> Root / checksum =>
for {
items <- backend.item.findByFileCollective(checksum, user.account.collective)
resp <- Ok(convert(items))
items <-
backend.itemSearch.findByFileCollective(checksum, user.account.collective)
resp <- Ok(convert(items))
} yield resp
}
@ -35,7 +36,7 @@ object CheckFileRoutes {
HttpRoutes.of {
case GET -> Root / Ident(id) / checksum =>
for {
items <- backend.item.findByFileSource(checksum, id)
items <- backend.itemSearch.findByFileSource(checksum, id)
resp <- Ok(convert(items))
} yield resp
}

View File

@ -4,7 +4,7 @@ import cats.effect._
import cats.implicits._
import docspell.backend.BackendApp
import docspell.backend.auth.AuthToken
import docspell.backend.ops.OItem.Batch
import docspell.backend.ops.OItemSearch.Batch
import docspell.common.{Ident, ItemState}
import org.http4s.HttpRoutes
import org.http4s.dsl.Http4sDsl
@ -34,7 +34,7 @@ object ItemRoutes {
_ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItems(
items <- backend.itemSearch.findItems(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -47,7 +47,7 @@ object ItemRoutes {
_ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItemsWithTags(
items <- backend.itemSearch.findItemsWithTags(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize)
)
@ -56,7 +56,7 @@ object ItemRoutes {
case GET -> Root / Ident(id) =>
for {
item <- backend.item.findItem(id, user.account.collective)
item <- backend.itemSearch.findItem(id, user.account.collective)
result = item.map(Conversions.mkItemDetail)
resp <-
result