Apply scalafmt to all files

This commit is contained in:
Eike Kettner 2019-12-30 21:44:13 +01:00
parent 57e274e2b0
commit fc3e22e399
133 changed files with 3003 additions and 2112 deletions

View File

@ -27,7 +27,11 @@ trait BackendApp[F[_]] {
object BackendApp { object BackendApp {
def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] = def create[F[_]: ConcurrentEffect](
cfg: Config,
store: Store[F],
httpClientEc: ExecutionContext
): Resource[F, BackendApp[F]] =
for { for {
queue <- JobQueue(store) queue <- JobQueue(store)
loginImpl <- Login[F](store) loginImpl <- Login[F](store)
@ -55,10 +59,12 @@ object BackendApp {
val item = itemImpl val item = itemImpl
} }
def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config def apply[F[_]: ConcurrentEffect: ContextShift](
, connectEC: ExecutionContext cfg: Config,
, httpClientEc: ExecutionContext connectEC: ExecutionContext,
, blocker: Blocker): Resource[F, BackendApp[F]] = httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, BackendApp[F]] =
for { for {
store <- Store.create(cfg.jdbc, connectEC, blocker) store <- Store.create(cfg.jdbc, connectEC, blocker)
backend <- create(cfg, store, httpClientEc) backend <- create(cfg, store, httpClientEc)

View File

@ -4,11 +4,7 @@ import docspell.backend.signup.{Config => SignupConfig}
import docspell.common.MimeType import docspell.common.MimeType
import docspell.store.JdbcConfig import docspell.store.JdbcConfig
case class Config( jdbc: JdbcConfig case class Config(jdbc: JdbcConfig, signup: SignupConfig, files: Config.Files) {}
, signup: SignupConfig
, files: Config.Files) {
}
object Config { object Config {

View File

@ -50,14 +50,13 @@ object AuthToken {
Left("Invalid authenticator") Left("Invalid authenticator")
} }
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = { def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for { for {
salt <- Common.genSaltString[F] salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "") cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key) sig = sign(cd, key)
} yield cd.copy(sig = sig) } yield cd.copy(sig = sig)
}
private def sign(cd: AuthToken, key: ByteVector): String = { private def sign(cd: AuthToken, key: ByteVector): String = {
val raw = cd.millis.toString + cd.account.asString + cd.salt val raw = cd.millis.toString + cd.account.asString + cd.salt

View File

@ -49,7 +49,8 @@ object Login {
def invalidTime: Result = InvalidTime def invalidTime: Result = InvalidTime
} }
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] { def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] =
Resource.pure(new Login[F] {
def loginSession(config: Config)(sessionKey: String): F[Result] = def loginSession(config: Config)(sessionKey: String): F[Result] =
AuthToken.fromString(sessionKey) match { AuthToken.fromString(sessionKey) match {
@ -61,10 +62,10 @@ object Login {
Result.invalidAuth.pure[F] Result.invalidAuth.pure[F]
} }
def loginUserPass(config: Config)(up: UserPass): F[Result] = { def loginUserPass(config: Config)(up: UserPass): F[Result] =
AccountId.parse(up.user) match { AccountId.parse(up.user) match {
case Right(acc) => case Right(acc) =>
val okResult= val okResult =
store.transact(RUser.updateLogin(acc)) *> store.transact(RUser.updateLogin(acc)) *>
AuthToken.user(acc, config.serverSecret).map(Result.ok) AuthToken.user(acc, config.serverSecret).map(Result.ok)
for { for {
@ -76,7 +77,6 @@ object Login {
case Left(_) => case Left(_) =>
Result.invalidAuth.pure[F] Result.invalidAuth.pure[F]
} }
}
private def check(given: String)(data: QLogin.Data): Boolean = { private def check(given: String)(data: QLogin.Data): Boolean = {
val collOk = data.collectiveState == CollectiveState.Active || val collOk = data.collectiveState == CollectiveState.Active ||

View File

@ -25,7 +25,11 @@ trait OCollective[F[_]] {
def insights(collective: Ident): F[InsightData] def insights(collective: Ident): F[InsightData]
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult]
} }
object OCollective { object OCollective {
@ -63,38 +67,46 @@ object OCollective {
} }
} }
def apply[F[_]: Effect](store: Store[F]): Resource[F, OCollective[F]] =
def apply[F[_]:Effect](store: Store[F]): Resource[F, OCollective[F]] =
Resource.pure(new OCollective[F] { Resource.pure(new OCollective[F] {
def find(name: Ident): F[Option[RCollective]] = def find(name: Ident): F[Option[RCollective]] =
store.transact(RCollective.findById(name)) store.transact(RCollective.findById(name))
def updateLanguage(collective: Ident, lang: Language): F[AddResult] = def updateLanguage(collective: Ident, lang: Language): F[AddResult] =
store.transact(RCollective.updateLanguage(collective, lang)). store
attempt.map(AddResult.fromUpdate) .transact(RCollective.updateLanguage(collective, lang))
.attempt
.map(AddResult.fromUpdate)
def listUser(collective: Ident): F[Vector[RUser]] = { def listUser(collective: Ident): F[Vector[RUser]] =
store.transact(RUser.findAll(collective, _.login)) store.transact(RUser.findAll(collective, _.login))
}
def add(s: RUser): F[AddResult] = def add(s: RUser): F[AddResult] =
store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login)) store.add(
RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))),
RUser.exists(s.login)
)
def update(s: RUser): F[AddResult] = def update(s: RUser): F[AddResult] =
store.add(RUser.update(s), RUser.exists(s.login)) store.add(RUser.update(s), RUser.exists(s.login))
def deleteUser(login: Ident, collective: Ident): F[AddResult] = def deleteUser(login: Ident, collective: Ident): F[AddResult] =
store.transact(RUser.delete(login, collective)). store.transact(RUser.delete(login, collective)).attempt.map(AddResult.fromUpdate)
attempt.map(AddResult.fromUpdate)
def insights(collective: Ident): F[InsightData] = def insights(collective: Ident): F[InsightData] =
store.transact(QCollective.getInsights(collective)) store.transact(QCollective.getInsights(collective))
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = { def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult] = {
val q = for { val q = for {
optUser <- RUser.findByAccount(accountId) optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) n <- check
.filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match { res = check match {
case Some(true) => case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed

View File

@ -17,7 +17,6 @@ trait OEquipment[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult] def delete(id: Ident, collective: Ident): F[AddResult]
} }
object OEquipment { object OEquipment {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] =
@ -46,9 +45,7 @@ object OEquipment {
n0 <- RItem.removeConcEquip(collective, id) n0 <- RItem.removeConcEquip(collective, id)
n1 <- REquipment.delete(id, collective) n1 <- REquipment.delete(id, collective)
} yield n0 + n1 } yield n0 + n1
store.transact(io). store.transact(io).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
}) })
} }

View File

@ -62,90 +62,98 @@ object OItem {
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte]) case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] =
Resource.pure(new OItem[F] { Resource.pure(new OItem[F] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] = def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store.transact(QItem.findItem(id)). store.transact(QItem.findItem(id)).map(opt => opt.flatMap(_.filterCollective(collective)))
map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = { def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] =
store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector
}
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = { def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store.transact(RAttachment.findByIdAndCollective(id, collective)). store
flatMap({ .transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) => case Some(ra) =>
store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last. store.bitpeace
map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))))) .get(ra.fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
)
)
case None => case None =>
(None: Option[AttachmentData[F]]).pure[F] (None: Option[AttachmentData[F]]).pure[F]
}) })
}
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = { def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for { val db = for {
cid <- RItem.getCollective(item) cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO] nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO] else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
} yield nd + ni } yield nd + ni
store.transact(db). store.transact(db).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] = def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] =
store.transact(RItem.updateDirection(item, collective, direction)). store
attempt. .transact(RItem.updateDirection(item, collective, direction))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] = def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrOrg(item, collective, org)). store.transact(RItem.updateCorrOrg(item, collective, org)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrPerson(item, collective, person)). store
attempt. .transact(RItem.updateCorrPerson(item, collective, person))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcPerson(item, collective, person)). store
attempt. .transact(RItem.updateConcPerson(item, collective, person))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] = def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcEquip(item, collective, equip)). store
attempt. .transact(RItem.updateConcEquip(item, collective, equip))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] = def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] =
store.transact(RItem.updateNotes(item, collective, notes)). store.transact(RItem.updateNotes(item, collective, notes)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setName(item: Ident, name: String, collective: Ident): F[AddResult] = def setName(item: Ident, name: String, collective: Ident): F[AddResult] =
store.transact(RItem.updateName(item, collective, name)). store.transact(RItem.updateName(item, collective, name)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] = def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] =
store.transact(RItem.updateStateForCollective(item, state, collective)). store
attempt. .transact(RItem.updateStateForCollective(item, state, collective))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDate(item, collective, date)). store.transact(RItem.updateDate(item, collective, date)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDueDate(item, collective, date)). store
attempt. .transact(RItem.updateDueDate(item, collective, date))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def delete(itemId: Ident, collective: Ident): F[Int] = def delete(itemId: Ident, collective: Ident): F[Int] =
QItem.delete(store)(itemId, collective) QItem.delete(store)(itemId, collective)

View File

@ -36,15 +36,19 @@ object OJob {
jobs.filter(_.job.state == JobState.Running) jobs.filter(_.job.state == JobState.Running)
} }
def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] = def apply[F[_]: ConcurrentEffect](
store: Store[F],
clientEC: ExecutionContext
): Resource[F, OJob[F]] =
Resource.pure(new OJob[F] { Resource.pure(new OJob[F] {
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = { def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)). store
map(t => JobDetail(t._1, t._2)). .transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong))
compile.toVector. .map(t => JobDetail(t._1, t._2))
map(CollectiveQueueState) .compile
} .toVector
.map(CollectiveQueueState)
def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = { def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = {
def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] = def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] =
@ -66,8 +70,9 @@ object OJob {
} }
def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] = def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] =
OJoex.cancelJob(job.id, worker, store, clientEC). OJoex
map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound) .cancelJob(job.id, worker, store, clientEC)
.map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
for { for {
tryDel <- store.transact(tryDelete) tryDel <- store.transact(tryDelete)

View File

@ -13,24 +13,32 @@ import scala.concurrent.ExecutionContext
import org.log4s._ import org.log4s._
object OJoex { object OJoex {
private [this] val logger = getLogger private[this] val logger = getLogger
def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = { def notifyAll[F[_]: ConcurrentEffect](
store: Store[F],
clientExecutionContext: ExecutionContext
): F[Unit] =
for { for {
nodes <- store.transact(RNode.findAll(NodeType.Joex)) nodes <- store.transact(RNode.findAll(NodeType.Joex))
_ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext)) _ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext))
} yield () } yield ()
}
def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] = def cancelJob[F[_]: ConcurrentEffect](
jobId: Ident,
worker: Ident,
store: Store[F],
clientEc: ExecutionContext
): F[Boolean] =
for { for {
node <- store.transact(RNode.findById(worker)) node <- store.transact(RNode.findById(worker))
cancel <- node.traverse(joexCancel(clientEc)(_, jobId)) cancel <- node.traverse(joexCancel(clientEc)(_, jobId))
} yield cancel.getOrElse(false) } yield cancel.getOrElse(false)
private def joexCancel[F[_]: ConcurrentEffect](
private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = { ec: ExecutionContext
val notifyUrl = node.url/"api"/"v1"/"job"/job.id/"cancel" )(node: RNode, job: Ident): F[Boolean] = {
val notifyUrl = node.url / "api" / "v1" / "job" / job.id / "cancel"
BlazeClientBuilder[F](ec).resource.use { client => BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => true) client.expect[String](req).map(_ => true)
@ -38,7 +46,7 @@ object OJoex {
} }
private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = { private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = {
val notifyUrl = node.url/"api"/"v1"/"notify" val notifyUrl = node.url / "api" / "v1" / "notify"
val execute = BlazeClientBuilder[F](ec).resource.use { client => val execute = BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => ()) client.expect[String](req).map(_ => ())

View File

@ -36,13 +36,15 @@ object OOrganization {
case class PersonAndContacts(person: RPerson, contacts: Seq[RContact]) case class PersonAndContacts(person: RPerson, contacts: Seq[RContact])
def apply[F[_] : Effect](store: Store[F]): Resource[F, OOrganization[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OOrganization[F]] =
Resource.pure(new OOrganization[F] { Resource.pure(new OOrganization[F] {
def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] = def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] =
store.transact(QOrganization.findOrgAndContact(account.collective, _.name)). store
map({ case (org, cont) => OrgAndContacts(org, cont) }). .transact(QOrganization.findOrgAndContact(account.collective, _.name))
compile.toVector .map({ case (org, cont) => OrgAndContacts(org, cont) })
.compile
.toVector
def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] = def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(ROrganization.findAllRef(account.collective, _.name)) store.transact(ROrganization.findAllRef(account.collective, _.name))
@ -54,9 +56,11 @@ object OOrganization {
QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store) QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store)
def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] = def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] =
store.transact(QOrganization.findPersonAndContact(account.collective, _.name)). store
map({ case (person, cont) => PersonAndContacts(person, cont) }). .transact(QOrganization.findPersonAndContact(account.collective, _.name))
compile.toVector .map({ case (person, cont) => PersonAndContacts(person, cont) })
.compile
.toVector
def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] = def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(RPerson.findAllRef(account.collective, _.name)) store.transact(RPerson.findAllRef(account.collective, _.name))
@ -68,14 +72,13 @@ object OOrganization {
QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store) QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store)
def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] = def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deleteOrg(orgId, collective)). store.transact(QOrganization.deleteOrg(orgId, collective)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def deletePerson(personId: Ident, collective: Ident): F[AddResult] = def deletePerson(personId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deletePerson(personId, collective)). store
attempt. .transact(QOrganization.deletePerson(personId, collective))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
}) })
} }

View File

@ -41,8 +41,6 @@ object OSource {
} }
def delete(id: Ident, collective: Ident): F[AddResult] = def delete(id: Ident, collective: Ident): F[AddResult] =
store.transact(RSource.delete(id, collective)). store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
}) })
} }

View File

@ -17,7 +17,6 @@ trait OTag[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult] def delete(id: Ident, collective: Ident): F[AddResult]
} }
object OTag { object OTag {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] =
@ -47,10 +46,7 @@ object OTag {
n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId)) n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective)) n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective))
} yield n0.getOrElse(0) + n1.getOrElse(0) } yield n0.getOrElse(0) + n1.getOrElse(0)
store.transact(io). store.transact(io).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
}) })
} }

View File

@ -22,17 +22,27 @@ trait OUpload[F[_]] {
} }
object OUpload { object OUpload {
private [this] val logger = getLogger private[this] val logger = getLogger
case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte]) case class File[F[_]](
name: Option[String],
advertisedMime: Option[MimeType],
data: Stream[F, Byte]
)
case class UploadMeta( direction: Option[Direction] case class UploadMeta(
, sourceAbbrev: String direction: Option[Direction],
, validFileTypes: Seq[MimeType]) sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
case class UploadData[F[_]]( multiple: Boolean case class UploadData[F[_]](
, meta: UploadMeta multiple: Boolean,
, files: Vector[File[F]], priority: Priority, tracker: Option[Ident]) meta: UploadMeta,
files: Vector[File[F]],
priority: Priority,
tracker: Option[Ident]
)
sealed trait UploadResult sealed trait UploadResult
object UploadResult { object UploadResult {
@ -41,22 +51,33 @@ object OUpload {
case object NoSource extends UploadResult case object NoSource extends UploadResult
} }
def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] = def apply[F[_]: ConcurrentEffect](
store: Store[F],
queue: JobQueue[F],
cfg: Config,
httpClientEC: ExecutionContext
): Resource[F, OUpload[F]] =
Resource.pure(new OUpload[F] { Resource.pure(new OUpload[F] {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = { def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] =
for { for {
files <- data.files.traverse(saveFile).map(_.flatten) files <- data.files.traverse(saveFile).map(_.flatten)
pred <- checkFileList(files) pred <- checkFileList(files)
lang <- store.transact(RCollective.findLanguage(account.collective)) lang <- store.transact(RCollective.findLanguage(account.collective))
meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes) meta = ProcessItemArgs.ProcessMeta(
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList)) account.collective,
lang.getOrElse(Language.German),
data.meta.direction,
data.meta.sourceAbbrev,
data.meta.validFileTypes
)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
else Vector(ProcessItemArgs(meta, files.toList))
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker)) job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
_ <- logger.fdebug(s"Storing jobs: $job") _ <- logger.fdebug(s"Storing jobs: $job")
res <- job.traverse(submitJobs) res <- job.traverse(submitJobs)
_ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)) _ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective))
} yield res.fold(identity, identity) } yield res.fold(identity, identity)
}
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] = def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for { for {
@ -67,30 +88,47 @@ object OUpload {
result <- accId.traverse(acc => submit(updata, acc)) result <- accId.traverse(acc => submit(updata, acc))
} yield result.fold(identity, identity) } yield result.fold(identity, identity)
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = { private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] =
for { for {
_ <- logger.fdebug(s"Storing jobs: $jobs") _ <- logger.fdebug(s"Storing jobs: $jobs")
_ <- queue.insertAll(jobs) _ <- queue.insertAll(jobs)
_ <- OJoex.notifyAll(store, httpClientEC) _ <- OJoex.notifyAll(store, httpClientEC)
} yield UploadResult.Success } yield UploadResult.Success
}
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = { private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
logger.finfo(s"Receiving file $file") *> logger.finfo(s"Receiving file $file") *>
store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None). store.bitpeace
compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt. .saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None)
map(_.fold(ex => { .compile
.lastOrError
.map(fm => Ident.unsafe(fm.id))
.attempt
.map(_.fold(ex => {
logger.warn(ex)(s"Could not store file for processing!") logger.warn(ex)(s"Could not store file for processing!")
None None
}, id => Some(ProcessItemArgs.File(file.name, id)))) }, id => Some(ProcessItemArgs.File(file.name, id))))
}
private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] = private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] =
Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(())) Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(()))
private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = { private def makeJobs(
args: Vector[ProcessItemArgs],
account: AccountId,
prio: Priority,
tracker: Option[Ident]
): F[Vector[RJob]] = {
def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob = def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob =
RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker) RJob.newJob(
id,
ProcessItemArgs.taskName,
account.collective,
arg,
arg.makeSubject,
now,
account.user,
prio,
tracker
)
for { for {
id <- Ident.randomId[F] id <- Ident.randomId[F]

View File

@ -21,19 +21,19 @@ trait OSignup[F[_]] {
object OSignup { object OSignup {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]:Effect](store: Store[F]): Resource[F, OSignup[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OSignup[F]] =
Resource.pure(new OSignup[F] { Resource.pure(new OSignup[F] {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = { def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
if (cfg.mode == Config.Mode.Invite) { if (cfg.mode == Config.Mode.Invite) {
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F] if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
NewInviteResult.passwordMismatch.pure[F]
else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id)) else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id))
} else { } else {
Effect[F].pure(NewInviteResult.invitationClosed) Effect[F].pure(NewInviteResult.invitationClosed)
} }
}
def register(cfg: Config)(data: RegisterData): F[SignupResult] = { def register(cfg: Config)(data: RegisterData): F[SignupResult] =
cfg.mode match { cfg.mode match {
case Config.Mode.Open => case Config.Mode.Open =>
addUser(data).map(SignupResult.fromAddResult) addUser(data).map(SignupResult.fromAddResult)
@ -61,7 +61,6 @@ object OSignup {
SignupResult.invalidInvitationKey.pure[F] SignupResult.invalidInvitationKey.pure[F]
} }
} }
}
private def retryInvite(res: SignupResult): Boolean = private def retryInvite(res: SignupResult): Boolean =
res match { res match {
@ -77,22 +76,30 @@ object OSignup {
false false
} }
private def addUser(data: RegisterData): F[AddResult] = { private def addUser(data: RegisterData): F[AddResult] = {
def toRecords: F[(RCollective, RUser)] = def toRecords: F[(RCollective, RUser)] =
for { for {
id2 <- Ident.randomId[F] id2 <- Ident.randomId[F]
now <- Timestamp.current[F] now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now) c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now) u = RUser(
id2,
data.login,
data.collName,
PasswordCrypt.crypt(data.password),
UserState.Active,
None,
0,
None,
now
)
} yield (c, u) } yield (c, u)
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = { def insert(coll: RCollective, user: RUser): ConnectionIO[Int] =
for { for {
n1 <- RCollective.insert(coll) n1 <- RCollective.insert(coll)
n2 <- RUser.insert(user) n2 <- RUser.insert(user)
} yield n1 + n2 } yield n1 + n2
}
def collectiveExists: ConnectionIO[Boolean] = def collectiveExists: ConnectionIO[Boolean] =
RCollective.existsById(data.collName) RCollective.existsById(data.collName)

View File

@ -2,9 +2,7 @@ package docspell.backend.signup
import docspell.store.AddResult import docspell.store.AddResult
sealed trait SignupResult { sealed trait SignupResult {}
}
object SignupResult { object SignupResult {

View File

@ -1,12 +1,14 @@
package docspell.common package docspell.common
case class Banner( component: String case class Banner(
, version: String component: String,
, gitHash: Option[String] version: String,
, jdbcUrl: LenientUri gitHash: Option[String],
, configFile: Option[String] jdbcUrl: LenientUri,
, appId: Ident configFile: Option[String],
, baseUrl: LenientUri) { appId: Ident,
baseUrl: LenientUri
) {
private val banner = private val banner =
"""______ _ _ """______ _ _
@ -21,12 +23,12 @@ case class Banner( component: String
def render(prefix: String): String = { def render(prefix: String): String = {
val text = banner.split('\n').toList ++ List( val text = banner.split('\n').toList ++ List(
s"<< $component >>" s"<< $component >>",
, s"Id: ${appId.id}" s"Id: ${appId.id}",
, s"Base-Url: ${baseUrl.asString}" s"Base-Url: ${baseUrl.asString}",
, s"Database: ${jdbcUrl.asString}" s"Database: ${jdbcUrl.asString}",
, s"Config: ${configFile.getOrElse("")}" s"Config: ${configFile.getOrElse("")}",
, "" ""
) )
text.map(line => s"$prefix $line").mkString("\n") text.map(line => s"$prefix $line").mkString("\n")

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe._ import io.circe._
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
case class IdRef(id: Ident, name: String) { case class IdRef(id: Ident, name: String) {}
}
object IdRef { object IdRef {

View File

@ -32,4 +32,3 @@ object ItemState {
implicit val jsonEncoder: Encoder[ItemState] = implicit val jsonEncoder: Encoder[ItemState] =
Encoder.encodeString.contramap(_.name) Encoder.encodeString.contramap(_.name)
} }

View File

@ -31,14 +31,12 @@ object Language {
def fromString(str: String): Either[String, Language] = { def fromString(str: String): Either[String, Language] = {
val lang = str.toLowerCase val lang = str.toLowerCase
all.find(_.allNames.contains(lang)). all.find(_.allNames.contains(lang)).toRight(s"Unsupported or invalid language: $str")
toRight(s"Unsupported or invalid language: $str")
} }
def unsafe(str: String): Language = def unsafe(str: String): Language =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
implicit val jsonDecoder: Decoder[Language] = implicit val jsonDecoder: Decoder[Language] =
Decoder.decodeString.emap(fromString) Decoder.decodeString.emap(fromString)
implicit val jsonEncoder: Encoder[Language] = implicit val jsonEncoder: Encoder[Language] =

View File

@ -12,7 +12,7 @@ case class MimeType(primary: String, sub: String) {
def matches(other: MimeType): Boolean = def matches(other: MimeType): Boolean =
primary == other.primary && primary == other.primary &&
(sub == other.sub || sub == "*" ) (sub == other.sub || sub == "*")
} }
object MimeType { object MimeType {
@ -26,9 +26,10 @@ object MimeType {
def image(sub: String): MimeType = def image(sub: String): MimeType =
MimeType("image", partFromString(sub).throwLeft) MimeType("image", partFromString(sub).throwLeft)
private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet private[this] val validChars: Set[Char] =
(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
def parse(str: String): Either[String, MimeType] = { def parse(str: String): Either[String, MimeType] =
str.indexOf('/') match { str.indexOf('/') match {
case -1 => Left(s"Invalid MIME type: $str") case -1 => Left(s"Invalid MIME type: $str")
case n => case n =>
@ -37,7 +38,6 @@ object MimeType {
sub <- partFromString(str.substring(n + 1)) sub <- partFromString(str.substring(n + 1))
} yield MimeType(prim.toLowerCase, sub.toLowerCase) } yield MimeType(prim.toLowerCase, sub.toLowerCase)
} }
}
def unsafe(str: String): MimeType = def unsafe(str: String): MimeType =
parse(str).throwLeft parse(str).throwLeft

View File

@ -2,6 +2,4 @@ package docspell.common
import java.time.LocalDate import java.time.LocalDate
case class NerDateLabel(date: LocalDate, label: NerLabel) { case class NerDateLabel(date: LocalDate, label: NerLabel) {}
}

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) { case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {}
}
object NerLabel { object NerLabel {
implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel] implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel]

View File

@ -4,7 +4,7 @@ import io.circe.{Decoder, Encoder}
final class Password(val pass: String) extends AnyVal { final class Password(val pass: String) extends AnyVal {
def isEmpty: Boolean= pass.isEmpty def isEmpty: Boolean = pass.isEmpty
override def toString: String = override def toString: String =
if (pass.isEmpty) "<empty>" else "***" if (pass.isEmpty) "<empty>" else "***"

View File

@ -16,7 +16,6 @@ object Priority {
case object Low extends Priority case object Low extends Priority
def fromString(str: String): Either[String, Priority] = def fromString(str: String): Either[String, Priority] =
str.toLowerCase match { str.toLowerCase match {
case "high" => Right(High) case "high" => Right(High)
@ -27,7 +26,6 @@ object Priority {
def unsafe(str: String): Priority = def unsafe(str: String): Priority =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
def fromInt(n: Int): Priority = def fromInt(n: Int): Priority =
if (n <= toInt(Low)) Low if (n <= toInt(Low)) Low
else High else High

View File

@ -6,14 +6,13 @@ import ProcessItemArgs._
case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
def makeSubject: String = { def makeSubject: String =
files.flatMap(_.name) match { files.flatMap(_.name) match {
case Nil => s"${meta.sourceAbbrev}: No files" case Nil => s"${meta.sourceAbbrev}: No files"
case n :: Nil => n case n :: Nil => n
case n1 :: n2 :: Nil => s"$n1, $n2" case n1 :: n2 :: Nil => s"$n1, $n2"
case _ => s"${files.size} files from ${meta.sourceAbbrev}" case _ => s"${files.size} files from ${meta.sourceAbbrev}"
} }
}
} }
@ -21,11 +20,13 @@ object ProcessItemArgs {
val taskName = Ident.unsafe("process-item") val taskName = Ident.unsafe("process-item")
case class ProcessMeta( collective: Ident case class ProcessMeta(
, language: Language collective: Ident,
, direction: Option[Direction] language: Language,
, sourceAbbrev: String direction: Option[Direction],
, validFileTypes: Seq[MimeType]) sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
object ProcessMeta { object ProcessMeta {
implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta] implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta]

View File

@ -30,8 +30,6 @@ object Timestamp {
def current[F[_]: Sync]: F[Timestamp] = def current[F[_]: Sync]: F[Timestamp] =
Sync[F].delay(Timestamp(Instant.now)) Sync[F].delay(Timestamp(Instant.now))
implicit val encodeTimestamp: Encoder[Timestamp] = implicit val encodeTimestamp: Encoder[Timestamp] =
BaseJsonCodecs.encodeInstantEpoch.contramap(_.value) BaseJsonCodecs.encodeInstantEpoch.contramap(_.value)

View File

@ -12,7 +12,6 @@ object UserState {
/** The user is blocked by an admin. */ /** The user is blocked by an admin. */
case object Disabled extends UserState case object Disabled extends UserState
def fromString(s: String): Either[String, UserState] = def fromString(s: String): Either[String, UserState] =
s.toLowerCase match { s.toLowerCase match {
case "active" => Right(Active) case "active" => Right(Active)

View File

@ -11,13 +11,18 @@ trait StreamSyntax {
implicit class StringStreamOps[F[_]](s: Stream[F, String]) { implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] = def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
s.fold("")(_ + _). s.fold("")(_ + _)
compile.last. .compile
map(optStr => for { .last
str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value")) .map(optStr =>
for {
str <- optStr
.map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying) json <- parse(str).leftMap(_.underlying)
value <- json.as[A] value <- json.as[A]
} yield value) } yield value
)
} }

View File

@ -5,17 +5,23 @@ import docspell.joex.scheduler.SchedulerConfig
import docspell.store.JdbcConfig import docspell.store.JdbcConfig
import docspell.text.ocr.{Config => OcrConfig} import docspell.text.ocr.{Config => OcrConfig}
case class Config(appId: Ident case class Config(
, baseUrl: LenientUri appId: Ident,
, bind: Config.Bind baseUrl: LenientUri,
, jdbc: JdbcConfig bind: Config.Bind,
, scheduler: SchedulerConfig jdbc: JdbcConfig,
, extraction: OcrConfig scheduler: SchedulerConfig,
extraction: OcrConfig
) )
object Config { object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") val postgres =
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
case class Bind(address: String, port: Int) case class Bind(address: String, port: Int)
} }

View File

@ -11,7 +11,6 @@ object ConfigFile {
def loadConfig: Config = def loadConfig: Config =
ConfigSource.default.at("docspell.joex").loadOrThrow[Config] ConfigSource.default.at("docspell.joex").loadOrThrow[Config]
object Implicits { object Implicits {
implicit val countingSchemeReader: ConfigReader[CountingScheme] = implicit val countingSchemeReader: ConfigReader[CountingScheme] =
ConfigReader[String].emap(reason(CountingScheme.readString)) ConfigReader[String].emap(reason(CountingScheme.readString))

View File

@ -12,11 +12,13 @@ import fs2.concurrent.SignallingRef
import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext
final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer](
, nodeOps: ONode[F] cfg: Config,
, store: Store[F] nodeOps: ONode[F],
, termSignal: SignallingRef[F, Boolean] store: Store[F],
, val scheduler: Scheduler[F]) extends JoexApp[F] { termSignal: SignallingRef[F, Boolean],
val scheduler: Scheduler[F]
) extends JoexApp[F] {
def init: F[Unit] = { def init: F[Unit] = {
val run = scheduler.start.compile.drain val run = scheduler.start.compile.drain
@ -40,16 +42,24 @@ final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Conf
object JoexAppImpl { object JoexAppImpl {
def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config def create[F[_]: ConcurrentEffect: ContextShift: Timer](
, termSignal: SignallingRef[F, Boolean] cfg: Config,
, connectEC: ExecutionContext termSignal: SignallingRef[F, Boolean],
, blocker: Blocker): Resource[F, JoexApp[F]] = connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, JoexApp[F]] =
for { for {
store <- Store.create(cfg.jdbc, connectEC, blocker) store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store) nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store). sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])). .withTask(
resource JobTask.json(
ProcessItemArgs.taskName,
ItemHandler[F](cfg.extraction),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown) appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR } yield appR

View File

@ -15,11 +15,17 @@ import scala.concurrent.ExecutionContext
object JoexServer { object JoexServer {
private case class App[F[_]](
httpApp: HttpApp[F],
termSig: SignallingRef[F, Boolean],
exitRef: Ref[F, ExitCode]
)
private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode]) def stream[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker) connectEC: ExecutionContext,
(implicit T: Timer[F]): Stream[F, Nothing] = { blocker: Blocker
)(implicit T: Timer[F]): Stream[F, Nothing] = {
val app = for { val app = for {
signal <- Resource.liftF(SignallingRef[F, Boolean](false)) signal <- Resource.liftF(SignallingRef[F, Boolean](false))
@ -36,13 +42,14 @@ object JoexServer {
} yield App(finalHttpApp, signal, exitCode) } yield App(finalHttpApp, signal, exitCode)
Stream
Stream.resource(app).flatMap(app => .resource(app)
BlazeServerBuilder[F]. .flatMap(app =>
bindHttp(cfg.bind.port, cfg.bind.address). BlazeServerBuilder[F]
withHttpApp(app.httpApp). .bindHttp(cfg.bind.port, cfg.bind.address)
withoutBanner. .withHttpApp(app.httpApp)
serveWhile(app.termSig, app.exitRef) .withoutBanner
.serveWhile(app.termSig, app.exitRef)
) )
}.drain }.drain

View File

@ -14,10 +14,12 @@ object Main extends IOApp {
private[this] val logger = getLogger private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor( val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))) Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc) val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService( val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))) Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))
)
def run(args: List[String]) = { def run(args: List[String]) = {
args match { args match {
@ -40,12 +42,15 @@ object Main extends IOApp {
} }
val cfg = ConfigFile.loadConfig val cfg = ConfigFile.loadConfig
val banner = Banner("JOEX" val banner = Banner(
, BuildInfo.version "JOEX",
, BuildInfo.gitHeadCommit BuildInfo.version,
, cfg.jdbc.url BuildInfo.gitHeadCommit,
, Option(System.getProperty("config.file")) cfg.jdbc.url,
, cfg.appId, cfg.baseUrl) Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}") logger.info(s"\n${banner.render("***>")}")
JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success) JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success)
} }

View File

@ -23,20 +23,28 @@ object CreateItem {
Task { ctx => Task { ctx =>
val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet
def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files). def fileMetas(itemId: Ident, now: Timestamp) =
flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))). Stream
collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }). .emits(ctx.args.files)
zipWithIndex. .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
evalMap({ case (f, index) => .collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f })
Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)) .zipWithIndex
}). .evalMap({
compile.toVector case (f, index) =>
Ident
.randomId[F]
.map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
})
.compile
.toVector
val item = RItem.newItem[F](ctx.args.meta.collective val item = RItem.newItem[F](
, ctx.args.makeSubject ctx.args.meta.collective,
, ctx.args.meta.sourceAbbrev ctx.args.makeSubject,
, ctx.args.meta.direction.getOrElse(Direction.Incoming) ctx.args.meta.sourceAbbrev,
, ItemState.Premature) ctx.args.meta.direction.getOrElse(Direction.Incoming),
ItemState.Premature
)
for { for {
_ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)") _ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)")
@ -56,16 +64,28 @@ object CreateItem {
Task { ctx => Task { ctx =>
for { for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F] _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F] _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))) else ().pure[F]
} yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)) rms <- cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
} yield cand.headOption.map(ri =>
ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)
)
} }
private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] = private def logDifferences[F[_]: Sync](
ctx: Context[F, ProcessItemArgs],
saved: Vector[RAttachment],
saveCount: Int
): F[Unit] =
if (ctx.args.files.size != saved.size) { if (ctx.args.files.size != saved.size) {
ctx.logger.warn(s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount") ctx.logger.warn(
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
)
} else { } else {
().pure[F] ().pure[F]
} }

View File

@ -19,45 +19,65 @@ object FindProposal {
def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
val rmas = data.metas.map(rm => val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
ctx.logger.info("Starting find-proposal") *> ctx.logger.info("Starting find-proposal") *>
rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))). rmas
flatMap(rmv => rmv.traverse(rm => .traverse(rm =>
processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))
)
.flatMap(rmv =>
rmv
.traverse(rm =>
ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *> ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *>
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))). ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))
map(_ => data.copy(metas = rmv))) )
.map(_ => data.copy(metas = rmv))
)
} }
def processAttachment[F[_]: Sync]( rm: RAttachmentMeta def processAttachment[F[_]: Sync](
, rd: Vector[NerDateLabel] rm: RAttachmentMeta,
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { rd: Vector[NerDateLabel],
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx)) val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx))
List(finder.find(rm.nerlabels), makeDateProposal(rd)). List(finder.find(rm.nerlabels), makeDateProposal(rd))
traverse(identity).map(MetaProposalList.flatten) .traverse(identity)
.map(MetaProposalList.flatten)
} }
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = { def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] =
Timestamp.current[F].map { now => Timestamp.current[F].map { now =>
val latestFirst = dates.sortWith(_.date isAfter _.date) val latestFirst = dates.sortWith((l1, l2) => l1.date.isAfter(l2.date))
val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate
val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate)) val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate))
val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate, val dueDates = MetaProposalList.fromSeq1(
after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) MetaProposalType.DueDate,
val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate, after.map(ndl =>
before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
val itemDates = MetaProposalList.fromSeq1(
MetaProposalType.DocDate,
before.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
MetaProposalList.flatten(Seq(dueDates, itemDates)) MetaProposalList.flatten(Seq(dueDates, itemDates))
} }
}
def removeDuplicates(labels: List[NerLabel]): List[NerLabel] = def removeDuplicates(labels: List[NerLabel]): List[NerLabel] =
labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) => labels
if (seen.contains(el.tag.name+el.label.toLowerCase)) (seen, result) .foldLeft((Set.empty[String], List.empty[NerLabel])) {
case ((seen, result), el) =>
if (seen.contains(el.tag.name + el.label.toLowerCase)) (seen, result)
else (seen + (el.tag.name + el.label.toLowerCase), el :: result) else (seen + (el.tag.name + el.label.toLowerCase), el :: result)
}._2.sortBy(_.startPosition) }
._2
.sortBy(_.startPosition)
trait Finder[F[_]] { self => trait Finder[F[_]] { self =>
def find(labels: Seq[NerLabel]): F[MetaProposalList] def find(labels: Seq[NerLabel]): F[MetaProposalList]
@ -80,12 +100,14 @@ object FindProposal {
else f.map(ml1 => ml0.fillEmptyFrom(ml1)) else f.map(ml1 => ml0.fillEmptyFrom(ml1))
}) })
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*) def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
(implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] = implicit F: FlatMap[F],
flatMap(res0 => { F2: Applicative[F]
): Finder[F] =
flatMap { res0 =>
if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0) if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0)
else f.map(res1 => res0.fillEmptyFrom(res1)) else f.map(res1 => res0.fillEmptyFrom(res1))
}) }
} }
object Finder { object Finder {
@ -102,7 +124,11 @@ object FindProposal {
labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten) labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten)
} }
private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { private def search[F[_]: Sync](
nt: NerLabel,
exact: Boolean,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val value = val value =
if (exact) normalizeSearchValue(nt.label) if (exact) normalizeSearchValue(nt.label)
else s"%${normalizeSearchValue(nt.label)}%" else s"%${normalizeSearchValue(nt.label)}%"
@ -110,42 +136,51 @@ object FindProposal {
if (exact) 2 else 5 if (exact) 2 else 5
if (value.length < minLength) { if (value.length < minLength) {
ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty) ctx.logger
} else nt.tag match { .debug(s"Skipping too small value '$value' (original '${nt.label}').")
.map(_ => MetaProposalList.empty)
} else
nt.tag match {
case NerTag.Organization => case NerTag.Organization =>
ctx.logger.debug(s"Looking for organizations: $value") *> ctx.logger.debug(s"Looking for organizations: $value") *>
ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)). ctx.store
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) .transact(ROrganization.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
case NerTag.Person => case NerTag.Person =>
val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)). val s1 = ctx.store
map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) .transact(RPerson.findLike(ctx.args.meta.collective, value, true))
val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)). .map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) val s2 = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking for persons: $value") *> (for { ctx.logger.debug(s"Looking for persons: $value") *> (for {
ml0 <- s1 ml0 <- s1
ml1 <- s2 ml1 <- s2
} yield ml0 |+| ml1) } yield ml0 |+| ml1)
case NerTag.Location => case NerTag.Location =>
ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'."). ctx.logger
map(_ => MetaProposalList.empty) .debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.")
.map(_ => MetaProposalList.empty)
case NerTag.Misc => case NerTag.Misc =>
ctx.logger.debug(s"Looking for equipments: $value") *> ctx.logger.debug(s"Looking for equipments: $value") *>
ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)). ctx.store
map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) .transact(REquipment.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
case NerTag.Email => case NerTag.Email =>
searchContact(nt, ContactKind.Email, value, ctx) searchContact(nt, ContactKind.Email, value, ctx)
case NerTag.Website => case NerTag.Website =>
if (!exact) { if (!exact) {
val searchString = Domain.domainFromUri(nt.label.toLowerCase). val searchString = Domain
toOption. .domainFromUri(nt.label.toLowerCase)
map(_.toPrimaryDomain.asString). .toOption
map(s => s"%$s%"). .map(_.toPrimaryDomain.asString)
getOrElse(value) .map(s => s"%$s%")
.getOrElse(value)
searchContact(nt, ContactKind.Website, searchString, ctx) searchContact(nt, ContactKind.Website, searchString, ctx)
} else { } else {
searchContact(nt, ContactKind.Website, value, ctx) searchContact(nt, ContactKind.Website, value, ctx)
@ -157,23 +192,28 @@ object FindProposal {
} }
} }
private def searchContact[F[_]: Sync]( nt: NerLabel private def searchContact[F[_]: Sync](
, kind: ContactKind nt: NerLabel,
, value: String kind: ContactKind,
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { value: String,
val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)). ctx: Context[F, ProcessItemArgs]
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) ): F[MetaProposalList] = {
val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)). val orgs = ctx.store
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) .transact(ROrganization.findLike(ctx.args.meta.collective, kind, value))
val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)). .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) val corrP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking with $kind: $value") *> ctx.logger.debug(s"Looking with $kind: $value") *>
List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten) List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten)
} }
// The backslash *must* be stripped from search strings. // The backslash *must* be stripped from search strings.
private [this] val invalidSearch = private[this] val invalidSearch =
"…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet "…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet
private def normalizeSearchValue(str: String): String = private def normalizeSearchValue(str: String): String =

View File

@ -4,10 +4,12 @@ import docspell.common.{Ident, NerDateLabel, NerLabel}
import docspell.joex.process.ItemData.AttachmentDates import docspell.joex.process.ItemData.AttachmentDates
import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
case class ItemData( item: RItem case class ItemData(
, attachments: Vector[RAttachment] item: RItem,
, metas: Vector[RAttachmentMeta] attachments: Vector[RAttachment],
, dateLabels: Vector[AttachmentDates]) { metas: Vector[RAttachmentMeta],
dateLabels: Vector[AttachmentDates]
) {
def findMeta(attachId: Ident): Option[RAttachmentMeta] = def findMeta(attachId: Ident): Option[RAttachmentMeta] =
metas.find(_.id == attachId) metas.find(_.id == attachId)
@ -16,7 +18,6 @@ case class ItemData( item: RItem
dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty) dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty)
} }
object ItemData { object ItemData {
case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) { case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) {

View File

@ -10,14 +10,13 @@ import docspell.text.ocr.{Config => OcrConfig}
object ItemHandler { object ItemHandler {
def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
logWarn("Now cancelling. Deleting potentially created data."). logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ => deleteByFileIds)
flatMap(_ => deleteByFileIds)
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] = def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] =
CreateItem[F]. CreateItem[F]
flatMap(itemStateTask(ItemState.Processing)). .flatMap(itemStateTask(ItemState.Processing))
flatMap(safeProcess[F](cfg)). .flatMap(safeProcess[F](cfg))
map(_ => ()) .map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] = def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx => Task { ctx =>
@ -30,22 +29,21 @@ object ItemHandler {
last = ctx.config.retries == current.getOrElse(0) last = ctx.config.retries == current.getOrElse(0)
} yield last } yield last
def safeProcess[F[_]: Sync: ContextShift](
def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] = cfg: OcrConfig
)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task(isLastRetry[F, ProcessItemArgs] _).flatMap { Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
case true => case true =>
ProcessItem[F](cfg)(data). ProcessItem[F](cfg)(data).attempt.flatMap({
attempt.flatMap({
case Right(d) => case Right(d) =>
Task.pure(d) Task.pure(d)
case Left(ex) => case Left(ex) =>
logWarn[F]("Processing failed on last retry. Creating item but without proposals."). logWarn[F]("Processing failed on last retry. Creating item but without proposals.")
flatMap(_ => itemStateTask(ItemState.Created)(data)). .flatMap(_ => itemStateTask(ItemState.Created)(data))
andThen(_ => Sync[F].raiseError(ex)) .andThen(_ => Sync[F].raiseError(ex))
}) })
case false => case false =>
ProcessItem[F](cfg)(data). ProcessItem[F](cfg)(data).flatMap(itemStateTask(ItemState.Created))
flatMap(itemStateTask(ItemState.Created))
} }
def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =

View File

@ -13,28 +13,40 @@ object LinkProposal {
val proposals = MetaProposalList.flatten(data.metas.map(_.proposals)) val proposals = MetaProposalList.flatten(data.metas.map(_.proposals))
ctx.logger.info(s"Starting linking proposals") *> ctx.logger.info(s"Starting linking proposals") *>
MetaProposalType.all. MetaProposalType.all
traverse(applyValue(data, proposals, ctx)). .traverse(applyValue(data, proposals, ctx))
map(result => ctx.logger.info(s"Results from proposal processing: $result")). .map(result => ctx.logger.info(s"Results from proposal processing: $result"))
map(_ => data) .map(_ => data)
} }
def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = { def applyValue[F[_]: Sync](
data: ItemData,
proposalList: MetaProposalList,
ctx: Context[F, ProcessItemArgs]
)(mpt: MetaProposalType): F[Result] =
proposalList.find(mpt) match { proposalList.find(mpt) match {
case None => case None =>
Result.noneFound(mpt).pure[F] Result.noneFound(mpt).pure[F]
case Some(a) if a.isSingleValue => case Some(a) if a.isSingleValue =>
ctx.logger.info(s"Found one candidate for ${a.proposalType}") *> ctx.logger.info(s"Found one candidate for ${a.proposalType}") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
map(_ => Result.single(mpt)) Result.single(mpt)
)
case Some(a) => case Some(a) =>
ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *> ctx.logger.info(
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first."
map(_ => Result.multiple(mpt)) ) *>
} setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.multiple(mpt)
)
} }
def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] = def setItemMeta[F[_]: Sync](
itemId: Ident,
ctx: Context[F, ProcessItemArgs],
mpt: MetaProposalType,
value: Ident
): F[Int] =
mpt match { mpt match {
case MetaProposalType.CorrOrg => case MetaProposalType.CorrOrg =>
ctx.logger.debug(s"Updating item organization with: ${value.id}") *> ctx.logger.debug(s"Updating item organization with: ${value.id}") *>
@ -54,7 +66,6 @@ object LinkProposal {
ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0) ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0)
} }
sealed trait Result { sealed trait Result {
def proposalType: MetaProposalType def proposalType: MetaProposalType
} }

View File

@ -7,13 +7,15 @@ import docspell.text.ocr.{Config => OcrConfig}
object ProcessItem { object ProcessItem {
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync: ContextShift](
TextExtraction(cfg, item). cfg: OcrConfig
flatMap(Task.setProgress(25)). )(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
flatMap(TextAnalysis[F]). TextExtraction(cfg, item)
flatMap(Task.setProgress(50)). .flatMap(Task.setProgress(25))
flatMap(FindProposal[F]). .flatMap(TextAnalysis[F])
flatMap(Task.setProgress(75)). .flatMap(Task.setProgress(50))
flatMap(LinkProposal[F]). .flatMap(FindProposal[F])
flatMap(Task.setProgress(99)) .flatMap(Task.setProgress(75))
.flatMap(LinkProposal[F])
.flatMap(Task.setProgress(99))
} }

View File

@ -8,7 +8,7 @@ import docspell.joex.scheduler.Task
import org.log4s._ import org.log4s._
object TestTasks { object TestTasks {
private [this] val logger = getLogger private[this] val logger = getLogger
def success[F[_]]: Task[F, ProcessItemArgs, Unit] = def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>
@ -17,9 +17,9 @@ object TestTasks {
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>
ctx.logger.info(s"Failing the task run :(").map(_ => ctx.logger
sys.error("Oh, cannot extract gold from this document") .info(s"Failing the task run :(")
) .map(_ => sys.error("Oh, cannot extract gold from this document"))
} }
def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
@ -34,6 +34,6 @@ object TestTasks {
ctx.logger.info("doing more things") ctx.logger.info("doing more things")
} }
private def sleep[F[_]:Sync](ms: Long): F[Unit] = private def sleep[F[_]: Sync](ms: Long): F[Unit] =
Sync[F].delay(Thread.sleep(ms)) Sync[F].delay(Thread.sleep(ms))
} }

View File

@ -19,21 +19,26 @@ object TextAnalysis {
s <- Duration.stopTime[F] s <- Duration.stopTime[F]
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))) _ <- t.traverse(m =>
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
)
e <- s e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") _ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector v = t.toVector
} yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2)) } yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2))
} }
def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = def annotateAttachment[F[_]: Sync](
lang: Language
)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
for { for {
list0 <- stanfordNer[F](lang, rm) list0 <- stanfordNer[F](lang, rm)
list1 <- contactNer[F](rm) list1 <- contactNer[F](rm)
dates <- dateNer[F](rm, lang) dates <- dateNer[F](rm, lang)
} yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates) } yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates)
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] =
Sync[F].delay {
rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty) rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty)
} }
@ -42,8 +47,10 @@ object TextAnalysis {
} }
def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay { def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay {
AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)) AttachmentDates(
rm,
rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)
)
} }
} }

View File

@ -11,7 +11,10 @@ import docspell.text.ocr.{TextExtract, Config => OcrConfig}
object TextExtraction { object TextExtraction {
def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig,
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
for { for {
_ <- ctx.logger.info("Starting text extraction") _ <- ctx.logger.info("Starting text extraction")
@ -24,22 +27,33 @@ object TextExtraction {
} yield item.copy(metas = txt) } yield item.copy(metas = txt)
} }
def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] = def extractTextToMeta[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: OcrConfig,
lang: Language
)(ra: RAttachment): F[RAttachmentMeta] =
for { for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}") _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}")
dst <- Duration.stopTime[F] dst <- Duration.stopTime[F]
txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra) txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra)
meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty)) meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty))
est <- dst est <- dst
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}") _ <- ctx.logger.debug(
s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}"
)
} yield meta } yield meta
def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = { def extractText[F[_]: Sync: ContextShift](
val data = store.bitpeace.get(ra.fileId.id). ocrConfig: OcrConfig,
unNoneTerminate. lang: Language,
through(store.bitpeace.fetchData2(RangeDef.all)) store: Store[F],
blocker: Blocker
)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.through(store.bitpeace.fetchData2(RangeDef.all))
TextExtract.extract(data, blocker, lang.iso3, ocrConfig). TextExtract.extract(data, blocker, lang.iso3, ocrConfig).compile.last
compile.last
} }
} }

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes { object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = { def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> (Root / "version") => case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version Ok(
, BuildInfo.builtAtMillis VersionInfo(
, BuildInfo.builtAtString BuildInfo.version,
, BuildInfo.gitHeadCommit.getOrElse("") BuildInfo.builtAtMillis,
, BuildInfo.gitDescribedVersion.getOrElse(""))) BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
} }
} }
} }

View File

@ -13,7 +13,7 @@ import org.http4s.dsl.Http4sDsl
object JoexRoutes { object JoexRoutes {
def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = { def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case POST -> Root / "notify" => case POST -> Root / "notify" =>
@ -31,7 +31,9 @@ object JoexRoutes {
case POST -> Root / "shutdownAndExit" => case POST -> Root / "shutdownAndExit" =>
for { for {
_ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown) _ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
resp <- Ok(BasicResult(true, "Shutdown initiated.")) resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp } yield resp
@ -52,7 +54,15 @@ object JoexRoutes {
} }
def mkJob(j: RJob): Job = def mkJob(j: RJob): Job =
Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch)) Job(
j.id,
j.subject,
j.submitted,
j.priority,
j.retries,
j.progress,
j.started.getOrElse(Timestamp.Epoch)
)
def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog = def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog =
JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList) JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList)

View File

@ -30,22 +30,26 @@ trait Context[F[_], A] { self =>
} }
object Context { object Context {
private [this] val log = getLogger private[this] val log = getLogger
def create[F[_]: Functor, A]( job: RJob def create[F[_]: Functor, A](
, arg: A job: RJob,
, config: SchedulerConfig arg: A,
, log: Logger[F] config: SchedulerConfig,
, store: Store[F] log: Logger[F],
, blocker: Blocker): Context[F, A] = store: Store[F],
blocker: Blocker
): Context[F, A] =
new ContextImpl(arg, log, store, blocker, config, job.id) new ContextImpl(arg, log, store, blocker, config, job.id)
def apply[F[_]: Concurrent, A]( job: RJob def apply[F[_]: Concurrent, A](
, arg: A job: RJob,
, config: SchedulerConfig arg: A,
, logSink: LogSink[F] config: SchedulerConfig,
, blocker: Blocker logSink: LogSink[F],
, store: Store[F]): F[Context[F, A]] = blocker: Blocker,
store: Store[F]
): F[Context[F, A]] =
for { for {
_ <- log.ftrace("Creating logger for task run") _ <- log.ftrace("Creating logger for task run")
logger <- Logger(job.id, job.info, config.logBufferSize, logSink) logger <- Logger(job.id, job.info, config.logBufferSize, logSink)
@ -53,13 +57,14 @@ object Context {
ctx = create[F, A](job, arg, config, logger, store, blocker) ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx } yield ctx
private final class ContextImpl[F[_]: Functor, A]( val args: A final private class ContextImpl[F[_]: Functor, A](
, val logger: Logger[F] val args: A,
, val store: Store[F] val logger: Logger[F],
, val blocker: Blocker val store: Store[F],
, val config: SchedulerConfig val blocker: Blocker,
, val jobId: Ident) val config: SchedulerConfig,
extends Context[F,A] { val jobId: Ident
) extends Context[F, A] {
def setProgress(percent: Int): F[Unit] = { def setProgress(percent: Int): F[Unit] = {
val pval = math.min(100, math.max(0, percent)) val pval = math.min(100, math.max(0, percent))

View File

@ -11,14 +11,13 @@ import docspell.common.Priority
*/ */
case class CountingScheme(high: Int, low: Int, counter: Int = 0) { case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
def nextPriority: (CountingScheme, Priority) = { def nextPriority: (CountingScheme, Priority) =
if (counter <= 0) (increment, Priority.High) if (counter <= 0) (increment, Priority.High)
else { else {
val rest = counter % (high + low) val rest = counter % (high + low)
if (rest < high) (increment, Priority.High) if (rest < high) (increment, Priority.High)
else (increment, Priority.Low) else (increment, Priority.Low)
} }
}
def increment: CountingScheme = def increment: CountingScheme =
copy(counter = counter + 1) copy(counter = counter + 1)
@ -32,8 +31,7 @@ object CountingScheme {
def readString(str: String): Either[String, CountingScheme] = def readString(str: String): Either[String, CountingScheme] =
str.split(',') match { str.split(',') match {
case Array(h, l) => case Array(h, l) =>
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)). Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage)
left.map(_.getMessage)
case _ => case _ =>
Left(s"Invalid counting scheme: $str") Left(s"Invalid counting scheme: $str")
} }

View File

@ -20,12 +20,15 @@ case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Tas
object JobTask { object JobTask {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit]) def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])(
(implicit D: Decoder[A]): JobTask[F] = { implicit D: Decoder[A]
): JobTask[F] = {
val convert: String => F[A] = val convert: String => F[A] =
str => str.parseJsonAs[A] match { str =>
str.parseJsonAs[A] match {
case Right(a) => a.pure[F] case Right(a) => a.pure[F]
case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) case Left(ex) =>
Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
} }
JobTask(name, task.contramap(convert), onCancel.contramap(convert)) JobTask(name, task.contramap(convert), onCancel.contramap(convert))

View File

@ -4,12 +4,14 @@ import cats.implicits._
import docspell.common._ import docspell.common._
import cats.effect.Sync import cats.effect.Sync
case class LogEvent( jobId: Ident case class LogEvent(
, jobInfo: String jobId: Ident,
, time: Timestamp jobInfo: String,
, level: LogLevel time: Timestamp,
, msg: String level: LogLevel,
, ex: Option[Throwable] = None) { msg: String,
ex: Option[Throwable] = None
) {
def logLine: String = def logLine: String =
s">>> ${time.asString} $level $jobInfo: $msg" s">>> ${time.asString} $level $jobInfo: $msg"
@ -21,5 +23,4 @@ object LogEvent {
def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] = def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] =
Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg))
} }

View File

@ -44,12 +44,22 @@ object LogSink {
LogSink(_.evalMap(e => logInternal(e))) LogSink(_.evalMap(e => logInternal(e)))
def db[F[_]: Sync](store: Store[F]): LogSink[F] = def db[F[_]: Sync](store: Store[F]): LogSink[F] =
LogSink(_.evalMap(ev => for { LogSink(
_.evalMap(ev =>
for {
id <- Ident.randomId[F] id <- Ident.randomId[F]
joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse("")) joblog = RJobLog(
id,
ev.jobId,
ev.level,
ev.time,
ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("")
)
_ <- logInternal(ev) _ <- logInternal(ev)
_ <- store.transact(RJobLog.insert(joblog)) _ <- store.transact(RJobLog.insert(joblog))
} yield ())) } yield ()
)
)
def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = { def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = {
val s: Stream[F, Pipe[F, LogEvent, Unit]] = val s: Stream[F, Pipe[F, LogEvent, Unit]] =

View File

@ -33,13 +33,21 @@ object Logger {
LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1) LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1)
def error(ex: Throwable)(msg: => String): F[Unit] = def error(ex: Throwable)(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1) LogEvent
.create[F](jobId, jobInfo, LogLevel.Error, msg)
.map(le => le.copy(ex = Some(ex)))
.flatMap(q.enqueue1)
def error(msg: => String): F[Unit] = def error(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1) LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1)
} }
def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] = def apply[F[_]: Concurrent](
jobId: Ident,
jobInfo: String,
bufferSize: Int,
sink: LogSink[F]
): F[Logger[F]] =
for { for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize) q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q) log = create(jobId, jobInfo, q)

View File

@ -7,13 +7,14 @@ import docspell.store.Store
import docspell.store.queue.JobQueue import docspell.store.queue.JobQueue
import fs2.concurrent.SignallingRef import fs2.concurrent.SignallingRef
case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( case class SchedulerBuilder[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig config: SchedulerConfig,
, tasks: JobTaskRegistry[F] tasks: JobTaskRegistry[F],
, store: Store[F] store: Store[F],
, blocker: Blocker blocker: Blocker,
, queue: Resource[F, JobQueue[F]] queue: Resource[F, JobQueue[F]],
, logSink: LogSink[F]) { logSink: LogSink[F]
) {
def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] =
copy(config = cfg) copy(config = cfg)
@ -33,7 +34,6 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] =
copy(logSink = sink) copy(logSink = sink)
def serve: Resource[F, Scheduler[F]] = def serve: Resource[F, Scheduler[F]] =
resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch)) resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch))
@ -45,22 +45,25 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
perms <- Resource.liftF(Semaphore(config.poolSize.toLong)) perms <- Resource.liftF(Semaphore(config.poolSize.toLong))
} yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms) } yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms)
scheduler.evalTap(_.init). scheduler.evalTap(_.init).map(s => s: Scheduler[F])
map(s => s: Scheduler[F])
} }
} }
object SchedulerBuilder { object SchedulerBuilder {
def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig def apply[F[_]: ConcurrentEffect: ContextShift](
, blocker: Blocker config: SchedulerConfig,
, store: Store[F]): SchedulerBuilder[F] = blocker: Blocker,
new SchedulerBuilder[F](config store: Store[F]
, JobTaskRegistry.empty[F] ): SchedulerBuilder[F] =
, store new SchedulerBuilder[F](
, blocker config,
, JobQueue(store) JobTaskRegistry.empty[F],
, LogSink.db[F](store)) store,
blocker,
JobQueue(store),
LogSink.db[F](store)
)
} }

View File

@ -2,24 +2,26 @@ package docspell.joex.scheduler
import docspell.common._ import docspell.common._
case class SchedulerConfig( name: Ident case class SchedulerConfig(
, poolSize: Int name: Ident,
, countingScheme: CountingScheme poolSize: Int,
, retries: Int countingScheme: CountingScheme,
, retryDelay: Duration retries: Int,
, logBufferSize: Int retryDelay: Duration,
, wakeupPeriod: Duration logBufferSize: Int,
) wakeupPeriod: Duration
)
object SchedulerConfig { object SchedulerConfig {
val default = SchedulerConfig( val default = SchedulerConfig(
name = Ident.unsafe("default-scheduler") name = Ident.unsafe("default-scheduler"),
, poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
, countingScheme = CountingScheme(2, 1) ,
, retries = 5 countingScheme = CountingScheme(2, 1),
, retryDelay = Duration.seconds(30) retries = 5,
, logBufferSize = 500 retryDelay = Duration.seconds(30),
, wakeupPeriod = Duration.minutes(10) logBufferSize = 500,
wakeupPeriod = Duration.minutes(10)
) )
} }

View File

@ -14,17 +14,19 @@ import SchedulerImpl._
import docspell.store.Store import docspell.store.Store
import docspell.store.queries.QJob import docspell.store.queries.QJob
final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
, blocker: Blocker val config: SchedulerConfig,
, queue: JobQueue[F] blocker: Blocker,
, tasks: JobTaskRegistry[F] queue: JobQueue[F],
, store: Store[F] tasks: JobTaskRegistry[F],
, logSink: LogSink[F] store: Store[F],
, state: SignallingRef[F, State[F]] logSink: LogSink[F],
, waiter: SignallingRef[F, Boolean] state: SignallingRef[F, State[F]],
, permits: Semaphore[F]) extends Scheduler[F] { waiter: SignallingRef[F, Boolean],
permits: Semaphore[F]
) extends Scheduler[F] {
private [this] val logger = getLogger private[this] val logger = getLogger
/** /**
* On startup, get all jobs in state running from this scheduler * On startup, get all jobs in state running from this scheduler
@ -34,8 +36,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.runningToWaiting(config.name, store) QJob.runningToWaiting(config.name, store)
def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] = def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] =
ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala). ConcurrentEffect[F].start(
evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain) Stream
.awakeEvery[F](config.wakeupPeriod.toScala)
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
.compile
.drain
)
def getRunning: F[Vector[RJob]] = def getRunning: F[Vector[RJob]] =
state.get.flatMap(s => QJob.findAll(s.getRunning, store)) state.get.flatMap(s => QJob.findAll(s.getRunning, store))
@ -51,21 +58,23 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def shutdown(cancelAll: Boolean): F[Unit] = { def shutdown(cancelAll: Boolean): F[Unit] = {
val doCancel = val doCancel =
state.get. state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ())
flatMap(_.cancelTokens.values.toList.traverse(identity)).
map(_ => ())
val runShutdown = val runShutdown =
state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F])
val wait = Stream.eval(runShutdown). val wait = Stream
evalMap(_ => logger.finfo("Scheduler is shutting down now.")). .eval(runShutdown)
flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))). .evalMap(_ => logger.finfo("Scheduler is shutting down now."))
flatMap(state => { .flatMap(_ =>
Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
)
.flatMap { state =>
if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running.")) if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running."))
else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++ else
Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
Stream.emit(state) Stream.emit(state)
}) }
(wait.drain ++ Stream.emit(())).compile.lastOrError (wait.drain ++ Stream.emit(())).compile.lastOrError
} }
@ -82,15 +91,24 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
_ <- logger.fdebug("New permit acquired") _ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest) down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay) else
queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") _ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release) _ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined } yield rjob.isDefined
Stream.eval(state.get.map(_.shutdownRequest)). Stream
evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]). .eval(state.get.map(_.shutdownRequest))
flatMap(if (_) Stream.empty else Stream.eval(body)). .evalTap(
flatMap({ if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
else ().pure[F]
)
.flatMap(if (_) Stream.empty else Stream.eval(body))
.flatMap({
case true => case true =>
mainLoop mainLoop
case false => case false =>
@ -103,7 +121,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def execute(job: RJob): F[Unit] = { def execute(job: RJob): F[Unit] = {
val task = for { val task = for {
jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}") jobtask <- tasks
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask } yield jobtask
task match { task match {
@ -123,7 +143,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onFinish(job: RJob, finalState: JobState): F[Unit] = def onFinish(job: RJob, finalState: JobState): F[Unit] =
for { for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)")) _ <- permits.release *> permits.available.flatMap(a =>
logger.fdebug(s"Permit released ($a free)")
)
_ <- state.modify(_.removeRunning(job)) _ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store) _ <- QJob.setFinalState(job.id, finalState, store)
} yield () } yield ()
@ -131,9 +153,14 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onStart(job: RJob): F[Unit] = def onStart(job: RJob): F[Unit] =
QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck
def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = { def wrapTask(
task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)). job: RJob,
mapF(_.attempt.flatMap({ task: Task[F, String, Unit],
ctx: Context[F, String]
): Task[F, String, Unit] =
task
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa))
.mapF(_.attempt.flatMap({
case Right(()) => case Right(()) =>
logger.info(s"Job execution successful: ${job.info}") logger.info(s"Job execution successful: ${job.info}")
ctx.logger.info("Job execution successful") *> ctx.logger.info("Job execution successful") *>
@ -148,16 +175,18 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.exceedsRetries(job.id, config.retries, store).flatMap { QJob.exceedsRetries(job.id, config.retries, store).flatMap {
case true => case true =>
logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded."). ctx.logger
map(_ => JobState.Failed: JobState) .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
.map(_ => JobState.Failed: JobState)
case false => case false =>
logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later."). ctx.logger
map(_ => JobState.Stuck: JobState) .error(ex)(s"Job ${job.info} execution failed. Retrying later.")
.map(_ => JobState.Stuck: JobState)
} }
} }
})). }))
mapF(_.attempt.flatMap { .mapF(_.attempt.flatMap {
case Right(jstate) => case Right(jstate) =>
onFinish(job, jstate) onFinish(job, jstate)
case Left(ex) => case Left(ex) =>
@ -166,13 +195,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
// since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways
onFinish(job, JobState.Stuck) onFinish(job, JobState.Stuck)
}) })
}
def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = { def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = {
val bfa = blocker.blockOn(code) val bfa = blocker.blockOn(code)
logger.fdebug(s"Forking job ${job.info}") *> logger.fdebug(s"Forking job ${job.info}") *>
ConcurrentEffect[F].start(bfa). ConcurrentEffect[F]
map(fiber => .start(bfa)
.map(fiber =>
logger.fdebug(s"Cancelling job ${job.info}") *> logger.fdebug(s"Cancelling job ${job.info}") *>
fiber.cancel *> fiber.cancel *>
onCancel.attempt.map({ onCancel.attempt.map({
@ -184,7 +213,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
state.modify(_.markCancelled(job)) *> state.modify(_.markCancelled(job)) *>
onFinish(job, JobState.Cancelled) *> onFinish(job, JobState.Cancelled) *>
ctx.logger.warn("Job has been cancelled.") *> ctx.logger.warn("Job has been cancelled.") *>
logger.fdebug(s"Job ${job.info} has been cancelled.")) logger.fdebug(s"Job ${job.info} has been cancelled.")
)
} }
} }
@ -193,10 +223,12 @@ object SchedulerImpl {
def emptyState[F[_]]: State[F] = def emptyState[F[_]]: State[F] =
State(Map.empty, Set.empty, Map.empty, false) State(Map.empty, Set.empty, Map.empty, false)
case class State[F[_]]( counters: Map[Ident, CountingScheme] case class State[F[_]](
, cancelled: Set[Ident] counters: Map[Ident, CountingScheme],
, cancelTokens: Map[Ident, CancelToken[F]] cancelled: Set[Ident],
, shutdownRequest: Boolean) { cancelTokens: Map[Ident, CancelToken[F]],
shutdownRequest: Boolean
) {
def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = {
val (cs, prio) = counters.getOrElse(group, initial).nextPriority val (cs, prio) = counters.getOrElse(group, initial).nextPriority

View File

@ -24,11 +24,11 @@ trait Task[F[_], A, B] {
def mapF[C](f: F[B] => F[C]): Task[F, A, C] = def mapF[C](f: F[B] => F[C]): Task[F, A, C] =
Task(Task.toKleisli(this).mapF(f)) Task(Task.toKleisli(this).mapF(f))
def attempt(implicit F: ApplicativeError[F,Throwable]): Task[F, A, Either[Throwable, B]] = def attempt(implicit F: ApplicativeError[F, Throwable]): Task[F, A, Either[Throwable, B]] =
mapF(_.attempt) mapF(_.attempt)
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { ctxc: Context[F, C] =>
ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
} }
} }
@ -46,7 +46,6 @@ object Task {
def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] =
c => k.run(c) c => k.run(c)
def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] =
Kleisli(t.run) Kleisli(t.run)

View File

@ -6,7 +6,7 @@ import minitest.SimpleTestSuite
object CountingSchemeSpec extends SimpleTestSuite { object CountingSchemeSpec extends SimpleTestSuite {
test("counting") { test("counting") {
val cs = CountingScheme(2,1) val cs = CountingScheme(2, 1)
val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2) val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2)
val expect = List(Priority.High, Priority.High, Priority.Low) val expect = List(Priority.High, Priority.High, Priority.Low)
assertEquals(list, expect ++ expect) assertEquals(list, expect ++ expect)

View File

@ -7,27 +7,37 @@ import docspell.backend.{Config => BackendConfig}
import docspell.common._ import docspell.common._
import scodec.bits.ByteVector import scodec.bits.ByteVector
case class Config(appName: String case class Config(
, appId: Ident appName: String,
, baseUrl: LenientUri appId: Ident,
, bind: Config.Bind baseUrl: LenientUri,
, backend: BackendConfig bind: Config.Bind,
, auth: Login.Config backend: BackendConfig,
auth: Login.Config
) )
object Config { object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") val postgres =
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
val default: Config = val default: Config =
Config("Docspell" Config(
, Ident.unsafe("restserver1") "Docspell",
, LenientUri.unsafe("http://localhost:7880") Ident.unsafe("restserver1"),
, Config.Bind("localhost", 7880) LenientUri.unsafe("http://localhost:7880"),
, BackendConfig(postgres Config.Bind("localhost", 7880),
, SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)) BackendConfig(
, BackendConfig.Files(512 * 1024, List(MimeType.pdf))) postgres,
, Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))) SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)),
BackendConfig.Files(512 * 1024, List(MimeType.pdf))
),
Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))
)
case class Bind(address: String, port: Int) case class Bind(address: String, port: Int)
} }

View File

@ -13,12 +13,14 @@ import org.log4s._
object Main extends IOApp { object Main extends IOApp {
private[this] val logger = getLogger private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool( val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
ThreadFactories.ofName("docspell-restserver-blocking"))) Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-restserver-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc) val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5, val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
ThreadFactories.ofName("docspell-dbconnect"))) Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-dbconnect"))
)
def run(args: List[String]) = { def run(args: List[String]) = {
args match { args match {
@ -41,12 +43,15 @@ object Main extends IOApp {
} }
val cfg = ConfigFile.loadConfig val cfg = ConfigFile.loadConfig
val banner = Banner("REST Server" val banner = Banner(
, BuildInfo.version "REST Server",
, BuildInfo.gitHeadCommit BuildInfo.version,
, cfg.backend.jdbc.url BuildInfo.gitHeadCommit,
, Option(System.getProperty("config.file")) cfg.backend.jdbc.url,
, cfg.appId, cfg.baseUrl) Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}") logger.info(s"\n${banner.render("***>")}")
RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success) RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success)
} }

View File

@ -7,7 +7,8 @@ import docspell.common.NodeType
import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F])
extends RestApp[F] {
def init: F[Unit] = def init: F[Unit] =
backend.node.register(config.appId, NodeType.Restserver, config.baseUrl) backend.node.register(config.appId, NodeType.Restserver, config.baseUrl)
@ -18,7 +19,12 @@ final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[
object RestAppImpl { object RestAppImpl {
def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] = def create[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, RestApp[F]] =
for { for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend) app = new RestAppImpl[F](cfg, backend)

View File

@ -15,8 +15,12 @@ import scala.concurrent.ExecutionContext
object RestServer { object RestServer {
def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker) def stream[F[_]: ConcurrentEffect](
(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = { cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
val app = for { val app = for {
restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker) restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker)
@ -24,8 +28,8 @@ object RestServer {
httpApp = Router( httpApp = Router(
"/api/info" -> routes.InfoRoutes(), "/api/info" -> routes.InfoRoutes(),
"/api/v1/open/" -> openRoutes(cfg, restApp), "/api/v1/open/" -> openRoutes(cfg, restApp),
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
token => securedRoutes(cfg, restApp, token) securedRoutes(cfg, restApp, token)
}, },
"/app/assets" -> WebjarRoutes.appRoutes[F](blocker), "/app/assets" -> WebjarRoutes.appRoutes[F](blocker),
"/app" -> TemplateRoutes[F](blocker, cfg) "/app" -> TemplateRoutes[F](blocker, cfg)
@ -35,16 +39,22 @@ object RestServer {
} yield finalHttpApp } yield finalHttpApp
Stream.resource(app).flatMap(httpApp => Stream
BlazeServerBuilder[F]. .resource(app)
bindHttp(cfg.bind.port, cfg.bind.address). .flatMap(httpApp =>
withHttpApp(httpApp). BlazeServerBuilder[F]
withoutBanner. .bindHttp(cfg.bind.port, cfg.bind.address)
serve) .withHttpApp(httpApp)
.withoutBanner
.serve
)
}.drain }.drain
def securedRoutes[F[_]: Effect](
def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] = cfg: Config,
restApp: RestApp[F],
token: AuthToken
): HttpRoutes[F] =
Router( Router(
"auth" -> LoginRoutes.session(restApp.backend.login, cfg), "auth" -> LoginRoutes.session(restApp.backend.login, cfg),
"tag" -> TagRoutes(restApp.backend, token), "tag" -> TagRoutes(restApp.backend, token),

View File

@ -13,8 +13,15 @@ case class CookieData(auth: AuthToken) {
def asCookie(cfg: Config): ResponseCookie = { def asCookie(cfg: Config): ResponseCookie = {
val domain = cfg.baseUrl.host val domain = cfg.baseUrl.host
val sec = cfg.baseUrl.scheme.exists(_.endsWith("s")) val sec = cfg.baseUrl.scheme.exists(_.endsWith("s"))
val path = cfg.baseUrl.path/"api"/"v1"/"sec" val path = cfg.baseUrl.path / "api" / "v1" / "sec"
ResponseCookie(CookieData.cookieName, asString, domain = domain, path = Some(path.asString), httpOnly = true, secure = sec) ResponseCookie(
CookieData.cookieName,
asString,
domain = domain,
path = Some(path.asString),
httpOnly = true,
secure = sec
)
} }
} }
object CookieData { object CookieData {
@ -22,18 +29,21 @@ object CookieData {
val headerName = "X-Docspell-Auth" val headerName = "X-Docspell-Auth"
def authenticator[F[_]](r: Request[F]): Either[String, String] = def authenticator[F[_]](r: Request[F]): Either[String, String] =
fromCookie(r) orElse fromHeader(r) fromCookie(r).orElse(fromHeader(r))
def fromCookie[F[_]](req: Request[F]): Either[String, String] = { def fromCookie[F[_]](req: Request[F]): Either[String, String] =
for { for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie") cookie <- header.values.toList
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
} yield cookie.content } yield cookie.content
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] = { def fromHeader[F[_]](req: Request[F]): Either[String, String] =
req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator") req.headers
} .get(CaseInsensitiveString(headerName))
.map(_.value)
.toRight("Couldn't find an authenticator")
def deleteCookie(cfg: Config): ResponseCookie = def deleteCookie(cfg: Config): ResponseCookie =
ResponseCookie( ResponseCookie(

View File

@ -24,31 +24,37 @@ trait Conversions {
// insights // insights
def mkItemInsights(d: InsightData): ItemInsights = def mkItemInsights(d: InsightData): ItemInsights =
ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))) ItemInsights(
d.incoming,
d.outgoing,
d.bytes,
TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))
)
// attachment meta // attachment meta
def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta =
AttachmentMeta(rm.content.getOrElse("") AttachmentMeta(
, rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)) rm.content.getOrElse(""),
, mkItemProposals(rm.proposals)) rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)),
mkItemProposals(rm.proposals)
)
// item proposal // item proposal
def mkItemProposals(ml: MetaProposalList): ItemProposals = { def mkItemProposals(ml: MetaProposalList): ItemProposals = {
def get(mpt: MetaProposalType) = def get(mpt: MetaProposalType) =
ml.find(mpt). ml.find(mpt).map(mp => mp.values.toList.map(_.ref).map(mkIdName)).getOrElse(Nil)
map(mp => mp.values.toList.map(_.ref).map(mkIdName)).
getOrElse(Nil)
def getDates(mpt: MetaProposalType): List[Timestamp] = def getDates(mpt: MetaProposalType): List[Timestamp] =
ml.find(mpt). ml.find(mpt)
map(mp => mp.values.toList. .map(mp =>
map(cand => cand.ref.id.id). mp.values.toList
flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption). .map(cand => cand.ref.id.id)
map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))). .flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption)
map(zdt => Timestamp(zdt.toInstant))). .map(_.atTime(12, 0).atZone(ZoneId.of("GMT")))
getOrElse(Nil). .map(zdt => Timestamp(zdt.toInstant))
distinct. )
take(5) .getOrElse(Nil)
.distinct
.take(5)
ItemProposals( ItemProposals(
corrOrg = get(MetaProposalType.CorrOrg), corrOrg = get(MetaProposalType.CorrOrg),
@ -62,23 +68,25 @@ trait Conversions {
// item detail // item detail
def mkItemDetail(data: OItem.ItemData): ItemDetail = def mkItemDetail(data: OItem.ItemData): ItemDetail =
ItemDetail(data.item.id ItemDetail(
, data.item.direction data.item.id,
, data.item.name data.item.direction,
, data.item.source data.item.name,
, data.item.state data.item.source,
, data.item.created data.item.state,
, data.item.updated data.item.created,
, data.item.itemDate data.item.updated,
, data.corrOrg.map(o => IdName(o.oid, o.name)) data.item.itemDate,
, data.corrPerson.map(p => IdName(p.pid, p.name)) data.corrOrg.map(o => IdName(o.oid, o.name)),
, data.concPerson.map(p => IdName(p.pid, p.name)) data.corrPerson.map(p => IdName(p.pid, p.name)),
, data.concEquip.map(e => IdName(e.eid, e.name)) data.concPerson.map(p => IdName(p.pid, p.name)),
, data.inReplyTo.map(mkIdName) data.concEquip.map(e => IdName(e.eid, e.name)),
, data.item.dueDate data.inReplyTo.map(mkIdName),
, data.item.notes data.item.dueDate,
, data.attachments.map((mkAttachment _).tupled).toList data.item.notes,
, data.tags.map(mkTag).toList) data.attachments.map((mkAttachment _).tupled).toList,
data.tags.map(mkTag).toList
)
def mkAttachment(ra: RAttachment, m: FileMeta): Attachment = def mkAttachment(ra: RAttachment, m: FileMeta): Attachment =
Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString)) Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString))
@ -86,20 +94,21 @@ trait Conversions {
// item list // item list
def mkQuery(m: ItemSearch, coll: Ident): OItem.Query = def mkQuery(m: ItemSearch, coll: Ident): OItem.Query =
OItem.Query(coll OItem.Query(
, m.name coll,
, if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed) m.name,
, m.direction if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed),
, m.corrPerson m.direction,
, m.corrOrg m.corrPerson,
, m.concPerson m.corrOrg,
, m.concEquip m.concPerson,
, m.tagsInclude.map(Ident.unsafe) m.concEquip,
, m.tagsExclude.map(Ident.unsafe) m.tagsInclude.map(Ident.unsafe),
, m.dateFrom m.tagsExclude.map(Ident.unsafe),
, m.dateUntil m.dateFrom,
, m.dueDateFrom m.dateUntil,
, m.dueDateUntil m.dueDateFrom,
m.dueDateUntil
) )
def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = { def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = {
@ -113,8 +122,20 @@ trait Conversions {
} }
def mkItemLight(i: OItem.ListItem): ItemLight = def mkItemLight(i: OItem.ListItem): ItemLight =
ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName), ItemLight(
i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount) i.id,
i.name,
i.state,
i.date,
i.dueDate,
i.source,
i.direction.name.some,
i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName),
i.concPerson.map(mkIdName),
i.concEquip.map(mkIdName),
i.fileCount
)
// job // job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
@ -128,46 +149,57 @@ trait Conversions {
val t2 = f(j2).getOrElse(Timestamp.Epoch) val t2 = f(j2).getOrElse(Timestamp.Epoch)
t1.value.isBefore(t2.value) t1.value.isBefore(t2.value)
} }
JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started)) JobQueueState(
, state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)) state.running.map(mkJobDetail).toList.sortWith(asc(_.started)),
, state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))) state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)),
state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))
)
} }
def mkJobDetail(jd: OJob.JobDetail): JobDetail = def mkJobDetail(jd: OJob.JobDetail): JobDetail =
JobDetail(jd.job.id JobDetail(
, jd.job.subject jd.job.id,
, jd.job.submitted jd.job.subject,
, jd.job.priority jd.job.submitted,
, jd.job.state jd.job.priority,
, jd.job.retries jd.job.state,
, jd.logs.map(mkJobLog).toList jd.job.retries,
, jd.job.progress jd.logs.map(mkJobLog).toList,
, jd.job.worker jd.job.progress,
, jd.job.started jd.job.worker,
, jd.job.finished) jd.job.started,
jd.job.finished
)
def mkJobLog(jl: RJobLog): JobLogEvent = def mkJobLog(jl: RJobLog): JobLogEvent =
JobLogEvent(jl.created, jl.level, jl.message) JobLogEvent(jl.created, jl.level, jl.message)
// upload // upload
def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = { def readMultipart[F[_]: Effect](
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = { mp: Multipart[F],
body.through(fs2.text.utf8Decode). logger: Logger,
parseJsonAs[ItemUploadMeta]. prio: Priority,
map(_.fold(ex => { validFileTypes: Seq[MimeType]
): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] =
body
.through(fs2.text.utf8Decode)
.parseJsonAs[ItemUploadMeta]
.map(_.fold(ex => {
logger.error(ex)("Reading upload metadata failed.") logger.error(ex)("Reading upload metadata failed.")
throw ex throw ex
}, identity)) }, identity))
}
val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")). val meta: F[(Boolean, UploadMeta)] = mp.parts
map(p => parseMeta(p.body)). .find(_.name.exists(_.equalsIgnoreCase("meta")))
map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))). .map(p => parseMeta(p.body))
getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) .map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes))))
.getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val files = mp.parts. val files = mp.parts
filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))). .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)) .map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
)
for { for {
metaData <- meta metaData <- meta
_ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData")) _ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData"))
@ -178,8 +210,14 @@ trait Conversions {
// organization and person // organization and person
def mkOrg(v: OOrganization.OrgAndContacts): Organization = { def mkOrg(v: OOrganization.OrgAndContacts): Organization = {
val ro = v.org val ro = v.org
Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), Organization(
v.contacts.map(mkContact).toList, ro.notes, ro.created) ro.oid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.created
)
} }
def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = { def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = {
@ -189,7 +227,17 @@ trait Conversions {
now <- Timestamp.current[F] now <- Timestamp.current[F]
oid <- Ident.randomId[F] oid <- Ident.randomId[F]
cont <- contacts(oid) cont <- contacts(oid)
org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now) org = ROrganization(
oid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
now
)
} yield OOrganization.OrgAndContacts(org, cont) } yield OOrganization.OrgAndContacts(org, cont)
} }
@ -198,14 +246,31 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, oid.some, None)) v.contacts.traverse(c => newContact(c, oid.some, None))
for { for {
cont <- contacts(v.id) cont <- contacts(v.id)
org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created) org = ROrganization(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.created
)
} yield OOrganization.OrgAndContacts(org, cont) } yield OOrganization.OrgAndContacts(org, cont)
} }
def mkPerson(v: OOrganization.PersonAndContacts): Person = { def mkPerson(v: OOrganization.PersonAndContacts): Person = {
val ro = v.person val ro = v.person
Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), Person(
v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created) ro.pid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.concerning,
ro.created
)
} }
def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = { def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = {
@ -215,7 +280,18 @@ trait Conversions {
now <- Timestamp.current[F] now <- Timestamp.current[F]
pid <- Ident.randomId[F] pid <- Ident.randomId[F]
cont <- contacts(pid) cont <- contacts(pid)
org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now) org = RPerson(
pid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
now
)
} yield OOrganization.PersonAndContacts(org, cont) } yield OOrganization.PersonAndContacts(org, cont)
} }
@ -224,7 +300,18 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, None, pid.some)) v.contacts.traverse(c => newContact(c, None, pid.some))
for { for {
cont <- contacts(v.id) cont <- contacts(v.id)
org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created) org = RPerson(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
v.created
)
} yield OOrganization.PersonAndContacts(org, cont) } yield OOrganization.PersonAndContacts(org, cont)
} }
@ -233,7 +320,8 @@ trait Conversions {
Contact(rc.contactId, rc.value, rc.kind) Contact(rc.contactId, rc.value, rc.kind)
def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] = def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] =
timeId.map { case (id, now) => timeId.map {
case (id, now) =>
RContact(id, c.value, c.kind, pid, oid, now) RContact(id, c.value, c.kind, pid, oid, now)
} }
@ -242,12 +330,33 @@ trait Conversions {
User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created) User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created)
def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] =
timeId.map { case (id, now) => timeId.map {
RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, 0, None, now) case (id, now) =>
RUser(
id,
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
0,
None,
now
)
} }
def changeUser(u: User, cid: Ident): RUser = def changeUser(u: User, cid: Ident): RUser =
RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created) RUser(
Ident.unsafe(""),
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
u.loginCount,
u.lastLogin,
u.created
)
// tags // tags
@ -255,21 +364,22 @@ trait Conversions {
Tag(rt.tagId, rt.name, rt.category, rt.created) Tag(rt.tagId, rt.name, rt.category, rt.created)
def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] =
timeId.map { case (id, now) => timeId.map {
case (id, now) =>
RTag(id, cid, t.name, t.category, now) RTag(id, cid, t.name, t.category, now)
} }
def changeTag(t: Tag, cid: Ident): RTag = def changeTag(t: Tag, cid: Ident): RTag =
RTag(t.id, cid, t.name, t.category, t.created) RTag(t.id, cid, t.name, t.category, t.created)
// sources // sources
def mkSource(s: RSource): Source = def mkSource(s: RSource): Source =
Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map({ case (id, now) => timeId.map({
case (id, now) =>
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
}) })
@ -281,7 +391,8 @@ trait Conversions {
Equipment(re.eid, re.name, re.created) Equipment(re.eid, re.name, re.created)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map({ case (id, now) => timeId.map({
case (id, now) =>
REquipment(id, cid, e.name, now) REquipment(id, cid, e.name, now)
}) })
@ -298,7 +409,8 @@ trait Conversions {
def basicResult(cr: JobCancelResult): BasicResult = def basicResult(cr: JobCancelResult): BasicResult =
cr match { cr match {
case JobCancelResult.JobNotFound => BasicResult(false, "Job not found") case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor") case JobCancelResult.CancelRequested =>
BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.") case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.")
} }
@ -317,7 +429,8 @@ trait Conversions {
def basicResult(cr: PassChangeResult): BasicResult = cr match { def basicResult(cr: PassChangeResult): BasicResult = cr match {
case PassChangeResult.Success => BasicResult(true, "Password changed.") case PassChangeResult.Success => BasicResult(true, "Password changed.")
case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.") case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.")
case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.") case PassChangeResult.PasswordMismatch =>
BasicResult(false, "The current password is incorrect.")
case PassChangeResult.UserNotFound => BasicResult(false, "User not found.") case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
} }

View File

@ -8,28 +8,26 @@ import org.http4s.dsl.Http4sDsl
trait ResponseGenerator[F[_]] { trait ResponseGenerator[F[_]] {
self: Http4sDsl[F] => self: Http4sDsl[F] =>
implicit final class EitherResponses[A, B](e: Either[A, B]) {
implicit final class EitherResponses[A,B](e: Either[A, B]) { def toResponse(headers: Header*)(
def toResponse(headers: Header*) implicit F: Applicative[F],
(implicit F: Applicative[F] w0: EntityEncoder[F, A],
, w0: EntityEncoder[F, A] w1: EntityEncoder[F, B]
, w1: EntityEncoder[F, B]): F[Response[F]] = ): F[Response[F]] =
e.fold( e.fold(
a => UnprocessableEntity(a), a => UnprocessableEntity(a),
b => Ok(b) b => Ok(b)
).map(_.withHeaders(headers: _*)) )
.map(_.withHeaders(headers: _*))
} }
implicit final class OptionResponse[A](o: Option[A]) { implicit final class OptionResponse[A](o: Option[A]) {
def toResponse(headers: Header*) def toResponse(
(implicit F: Applicative[F] headers: Header*
, w0: EntityEncoder[F, A]): F[Response[F]] = )(implicit F: Applicative[F], w0: EntityEncoder[F, A]): F[Response[F]] =
o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*)) o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*))
} }
} }
object ResponseGenerator { object ResponseGenerator {}
}

View File

@ -18,17 +18,18 @@ import org.http4s.headers.ETag.EntityTag
object AttachmentRoutes { object AttachmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = { def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = {
val mt = MediaType.unsafeParse(data.meta.mimetype.asString) val mt = MediaType.unsafeParse(data.meta.mimetype.asString)
val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length) val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length)
val eTag: Header = ETag(data.meta.checksum) val eTag: Header = ETag(data.meta.checksum)
val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse(""))) val disp: Header =
Ok(data.data.take(data.meta.length)). `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
map(r => r.withContentType(`Content-Type`(mt)). Ok(data.data.take(data.meta.length)).map(r =>
withHeaders(cntLen, eTag, disp)) r.withContentType(`Content-Type`(mt)).withHeaders(cntLen, eTag, disp)
)
} }
HttpRoutes.of { HttpRoutes.of {
@ -38,7 +39,8 @@ object AttachmentRoutes {
inm = req.headers.get(`If-None-Match`).flatMap(_.tags) inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData, inm) matches = matchETag(fileData, inm)
resp <- if (matches) NotModified() resp <- if (matches) NotModified()
else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found"))) else
fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case GET -> Root / Ident(id) / "meta" => case GET -> Root / Ident(id) / "meta" =>
@ -50,8 +52,10 @@ object AttachmentRoutes {
} }
} }
private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]] private def matchETag[F[_]](
, noneMatch: Option[NonEmptyList[EntityTag]]): Boolean = fileData: Option[OItem.AttachmentData[F]],
noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean =
(fileData, noneMatch) match { (fileData, noneMatch) match {
case (Some(fd), Some(nm)) => case (Some(fd), Some(nm)) =>
fd.meta.checksum == nm.head.tag fd.meta.checksum == nm.head.tag

View File

@ -12,14 +12,17 @@ import org.http4s.server._
object Authenticate { object Authenticate {
def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] = def authenticateRequest[F[_]: Effect](
auth: String => F[Login.Result]
)(req: Request[F]): F[Login.Result] =
CookieData.authenticator(req) match { CookieData.authenticator(req) match {
case Right(str) => auth(str) case Right(str) => auth(str)
case Left(_) => Login.Result.invalidAuth.pure[F] case Left(_) => Login.Result.invalidAuth.pure[F]
} }
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = { pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {} val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._ import dsl._
@ -34,7 +37,9 @@ object Authenticate {
middleware(AuthedRoutes.of(pf)) middleware(AuthedRoutes.of(pf))
} }
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = { def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(
f: AuthToken => HttpRoutes[F]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {} val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._ import dsl._
@ -49,6 +54,8 @@ object Authenticate {
middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req))) middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req)))
} }
private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] = private def getUser[F[_]: Effect](
auth: String => F[Login.Result]
): Kleisli[F, Request[F], Either[String, AuthToken]] =
Kleisli(r => authenticateRequest(auth)(r).map(_.toEither)) Kleisli(r => authenticateRequest(auth)(r).map(_.toEither))
} }

View File

@ -25,7 +25,7 @@ object CollectiveRoutes {
resp <- Ok(Conversions.mkItemInsights(ins)) resp <- Ok(Conversions.mkItemInsights(ins))
} yield resp } yield resp
case req@POST -> Root / "settings" => case req @ POST -> Root / "settings" =>
for { for {
settings <- req.as[CollectiveSettings] settings <- req.as[CollectiveSettings]
res <- backend.collective.updateLanguage(user.account.collective, settings.language) res <- backend.collective.updateLanguage(user.account.collective, settings.language)

View File

@ -15,7 +15,7 @@ import org.http4s.dsl.Http4sDsl
object EquipmentRoutes { object EquipmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes { object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = { def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> (Root / "version") => case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version Ok(
, BuildInfo.builtAtMillis VersionInfo(
, BuildInfo.builtAtString BuildInfo.version,
, BuildInfo.gitHeadCommit.getOrElse("") BuildInfo.builtAtMillis,
, BuildInfo.gitDescribedVersion.getOrElse(""))) BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
} }
} }
} }

View File

@ -18,7 +18,7 @@ object ItemRoutes {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
@ -51,63 +51,63 @@ object ItemRoutes {
resp <- Ok(Conversions.basicResult(res, "Item back to created.")) resp <- Ok(Conversions.basicResult(res, "Item back to created."))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "tags" => case req @ POST -> Root / Ident(id) / "tags" =>
for { for {
tags <- req.as[ReferenceList].map(_.items) tags <- req.as[ReferenceList].map(_.items)
res <- backend.item.setTags(id, tags.map(_.id), user.account.collective) res <- backend.item.setTags(id, tags.map(_.id), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Tags updated")) resp <- Ok(Conversions.basicResult(res, "Tags updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "direction" => case req @ POST -> Root / Ident(id) / "direction" =>
for { for {
dir <- req.as[DirectionValue] dir <- req.as[DirectionValue]
res <- backend.item.setDirection(id, dir.direction, user.account.collective) res <- backend.item.setDirection(id, dir.direction, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Direction updated")) resp <- Ok(Conversions.basicResult(res, "Direction updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "corrOrg" => case req @ POST -> Root / Ident(id) / "corrOrg" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setCorrOrg(id, idref.id, user.account.collective) res <- backend.item.setCorrOrg(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated")) resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "corrPerson" => case req @ POST -> Root / Ident(id) / "corrPerson" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setCorrPerson(id, idref.id, user.account.collective) res <- backend.item.setCorrPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent person updated")) resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "concPerson" => case req @ POST -> Root / Ident(id) / "concPerson" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setConcPerson(id, idref.id, user.account.collective) res <- backend.item.setConcPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned person updated")) resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "concEquipment" => case req @ POST -> Root / Ident(id) / "concEquipment" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setConcEquip(id, idref.id, user.account.collective) res <- backend.item.setConcEquip(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "notes" => case req @ POST -> Root / Ident(id) / "notes" =>
for { for {
text <- req.as[OptionalText] text <- req.as[OptionalText]
res <- backend.item.setNotes(id, text.text, user.account.collective) res <- backend.item.setNotes(id, text.text, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "name" => case req @ POST -> Root / Ident(id) / "name" =>
for { for {
text <- req.as[OptionalText] text <- req.as[OptionalText]
res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective) res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "duedate" => case req @ POST -> Root / Ident(id) / "duedate" =>
for { for {
date <- req.as[OptionalDate] date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item due date to ${date.date}") _ <- logger.fdebug(s"Setting item due date to ${date.date}")
@ -115,7 +115,7 @@ object ItemRoutes {
resp <- Ok(Conversions.basicResult(res, "Item due date updated")) resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "date" => case req @ POST -> Root / Ident(id) / "date" =>
for { for {
date <- req.as[OptionalDate] date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item date to ${date.date}") _ <- logger.fdebug(s"Setting item date to ${date.date}")

View File

@ -18,7 +18,7 @@ object LoginRoutes {
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case req@POST -> Root / "login" => case req @ POST -> Root / "login" =>
for { for {
up <- req.as[UserPass] up <- req.as[UserPass]
res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password)) res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password))
@ -33,22 +33,36 @@ object LoginRoutes {
HttpRoutes.of[F] { HttpRoutes.of[F] {
case req @ POST -> Root / "session" => case req @ POST -> Root / "session" =>
Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req). Authenticate
flatMap(res => makeResponse(dsl, cfg, res, "")) .authenticateRequest(S.loginSession(cfg.auth))(req)
.flatMap(res => makeResponse(dsl, cfg, res, ""))
case POST -> Root / "logout" => case POST -> Root / "logout" =>
Ok().map(_.addCookie(CookieData.deleteCookie(cfg))) Ok().map(_.addCookie(CookieData.deleteCookie(cfg)))
} }
} }
def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = { def makeResponse[F[_]: Effect](
dsl: Http4sDsl[F],
cfg: Config,
res: Login.Result,
account: String
): F[Response[F]] = {
import dsl._ import dsl._
res match { res match {
case Login.Result.Ok(token) => case Login.Result.Ok(token) =>
for { for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply) cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)). resp <- Ok(
map(_.addCookie(cd.asCookie(cfg))) AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
} yield resp } yield resp
case _ => case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L)) Ok(AuthResult("", account, false, "Login failed.", None, 0L))

View File

@ -16,7 +16,7 @@ import org.http4s.dsl.Http4sDsl
object OrganizationRoutes { object OrganizationRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {

View File

@ -6,9 +6,10 @@ import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher
object ParamDecoder { object ParamDecoder {
implicit val booleanDecoder: QueryParamDecoder[Boolean] = implicit val booleanDecoder: QueryParamDecoder[Boolean] =
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean") QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_.equalsIgnoreCase("true")))(
"Boolean"
)
object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full") object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full")
} }

View File

@ -19,7 +19,7 @@ object PersonRoutes {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {

View File

@ -29,7 +29,7 @@ object RegisterRoutes {
resp <- Ok(convert(res)) resp <- Ok(convert(res))
} yield resp } yield resp
case req@ POST -> Root / "newinvite" => case req @ POST -> Root / "newinvite" =>
for { for {
data <- req.as[GenInvite] data <- req.as[GenInvite]
res <- backend.signup.newInvite(cfg.backend.signup)(data.password) res <- backend.signup.newInvite(cfg.backend.signup)(data.password)
@ -47,7 +47,6 @@ object RegisterRoutes {
InviteResult(false, "Password is invalid.", None) InviteResult(false, "Password is invalid.", None)
} }
def convert(r: SignupResult): BasicResult = r match { def convert(r: SignupResult): BasicResult = r match {
case SignupResult.CollectiveExists => case SignupResult.CollectiveExists =>
BasicResult(false, "A collective with this name already exists.") BasicResult(false, "A collective with this name already exists.")
@ -62,7 +61,6 @@ object RegisterRoutes {
BasicResult(true, "Signup successful") BasicResult(true, "Signup successful")
} }
def convert(r: Registration): RegisterData = def convert(r: Registration): RegisterData =
RegisterData(r.collectiveName, r.login, r.password, r.invite) RegisterData(r.collectiveName, r.login, r.password, r.invite)
} }

View File

@ -26,7 +26,12 @@ object UploadRoutes {
case req @ POST -> Root / "item" => case req @ POST -> Root / "item" =>
for { for {
multipart <- req.as[Multipart[F]] multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes) updata <- readMultipart(
multipart,
logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
result <- backend.upload.submit(updata, user.account) result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result)) res <- Ok(basicResult(result))
} yield res } yield res
@ -39,7 +44,7 @@ object UploadRoutes {
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "item" / Ident(id)=> case req @ POST -> Root / "item" / Ident(id) =>
for { for {
multipart <- req.as[Multipart[F]] multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes) updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes)

View File

@ -23,7 +23,11 @@ object UserRoutes {
case req @ POST -> Root / "changePassword" => case req @ POST -> Root / "changePassword" =>
for { for {
data <- req.as[PasswordChange] data <- req.as[PasswordChange]
res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword) res <- backend.collective.changePassword(
user.account,
data.currentPassword,
data.newPassword
)
resp <- Ok(basicResult(res)) resp <- Ok(basicResult(res))
} yield resp } yield resp

View File

@ -8,14 +8,21 @@ import docspell.backend.signup.{Config => SignupConfig}
import yamusca.imports._ import yamusca.imports._
import yamusca.implicits._ import yamusca.implicits._
case class Flags( appName: String case class Flags(
, baseUrl: LenientUri appName: String,
, signupMode: SignupConfig.Mode baseUrl: LenientUri,
, docspellAssetPath: String) signupMode: SignupConfig.Mode,
docspellAssetPath: String
)
object Flags { object Flags {
def apply(cfg: Config): Flags = def apply(cfg: Config): Flags =
Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode, s"assets/docspell-webapp/${BuildInfo.version}") Flags(
cfg.appName,
cfg.baseUrl,
cfg.backend.signup.mode,
s"assets/docspell-webapp/${BuildInfo.version}"
)
implicit val jsonEncoder: Encoder[Flags] = implicit val jsonEncoder: Encoder[Flags] =
deriveEncoder[Flags] deriveEncoder[Flags]

View File

@ -21,11 +21,13 @@ object TemplateRoutes {
val `text/html` = new MediaType("text", "html") val `text/html` = new MediaType("text", "html")
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit C: ContextShift[F]): HttpRoutes[F] = { def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(
implicit C: ContextShift[F]
): HttpRoutes[F] = {
val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker))) val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker)))
val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker))) val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker)))
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> Root / "index.html" => case GET -> Root / "index.html" =>
@ -41,20 +43,21 @@ object TemplateRoutes {
} }
} }
def loadResource[F[_]: Sync](name: String): F[URL] = { def loadResource[F[_]: Sync](name: String): F[URL] =
Option(getClass.getResource(name)) match { Option(getClass.getResource(name)) match {
case None => case None =>
Sync[F].raiseError(new Exception("Unknown resource: "+ name)) Sync[F].raiseError(new Exception("Unknown resource: " + name))
case Some(r) => case Some(r) =>
r.pure[F] r.pure[F]
} }
}
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] = def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] =
Stream.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())). Stream
flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)). .bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
through(text.utf8Decode). .flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false))
compile.fold("")(_ + _) .through(text.utf8Decode)
.compile
.fold("")(_ + _)
def parseTemplate[F[_]: Sync](str: String): F[Template] = def parseTemplate[F[_]: Sync](str: String): F[Template] =
Sync[F].delay { Sync[F].delay {
@ -64,47 +67,54 @@ object TemplateRoutes {
} }
} }
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[Template] = { def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)). implicit C: ContextShift[F]
map(t => { ): F[Template] =
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
logger.info(s"Compiled template $url") logger.info(s"Compiled template $url")
t t
})
} }
case class DocData(swaggerRoot: String, openapiSpec: String) case class DocData(swaggerRoot: String, openapiSpec: String)
object DocData { object DocData {
def apply(): DocData = def apply(): DocData =
DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml") DocData(
"/app/assets" + Webjars.swaggerui,
s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml"
)
implicit def yamuscaValueConverter: ValueConverter[DocData] = implicit def yamuscaValueConverter: ValueConverter[DocData] =
ValueConverter.deriveConverter[DocData] ValueConverter.deriveConverter[DocData]
} }
case class IndexData(flags: Flags case class IndexData(
, cssUrls: Seq[String] flags: Flags,
, jsUrls: Seq[String] cssUrls: Seq[String],
, faviconBase: String jsUrls: Seq[String],
, appExtraJs: String faviconBase: String,
, flagsJson: String) appExtraJs: String,
flagsJson: String
)
object IndexData { object IndexData {
def apply(cfg: Config): IndexData = def apply(cfg: Config): IndexData =
IndexData(Flags(cfg) IndexData(
, Seq( Flags(cfg),
Seq(
"/app/assets" + Webjars.semanticui + "/semantic.min.css", "/app/assets" + Webjars.semanticui + "/semantic.min.css",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css" s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css"
) ),
, Seq( Seq(
"/app/assets" + Webjars.jquery + "/jquery.min.js", "/app/assets" + Webjars.jquery + "/jquery.min.js",
"/app/assets" + Webjars.semanticui + "/semantic.min.js", "/app/assets" + Webjars.semanticui + "/semantic.min.js",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js" s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js"
),
s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js",
Flags(cfg).asJson.spaces2
) )
, s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon"
, s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js"
, Flags(cfg).asJson.spaces2 )
implicit def yamuscaValueConverter: ValueConverter[IndexData] = implicit def yamuscaValueConverter: ValueConverter[IndexData] =
ValueConverter.deriveConverter[IndexData] ValueConverter.deriveConverter[IndexData]
@ -116,10 +126,10 @@ object TemplateRoutes {
Option(ref.get) match { Option(ref.get) match {
case Some(a) => a.pure[F] case Some(a) => a.pure[F]
case None => case None =>
fa.map(a => { fa.map { a =>
ref.set(a) ref.set(a)
a a
}) }
} }
} }
} }

View File

@ -9,7 +9,7 @@ import org.http4s.server.staticcontent.WebjarService.{WebjarAsset, Config => Web
object WebjarRoutes { object WebjarRoutes {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = { def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] =
webjarService( webjarService(
WebjarConfig( WebjarConfig(
filter = assetFilter, filter = assetFilter,
@ -17,10 +17,23 @@ object WebjarRoutes {
cacheStrategy = NoopCacheStrategy[F] cacheStrategy = NoopCacheStrategy[F]
) )
) )
}
def assetFilter(asset: WebjarAsset): Boolean = def assetFilter(asset: WebjarAsset): Boolean =
List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml"). List(
exists(e => asset.asset.endsWith(e)) ".js",
".css",
".html",
".json",
".jpg",
".png",
".eot",
".woff",
".woff2",
".svg",
".otf",
".ttf",
".yml",
".xml"
).exists(e => asset.asset.endsWith(e))
} }

View File

@ -2,10 +2,7 @@ package docspell.store
import docspell.common.LenientUri import docspell.common.LenientUri
case class JdbcConfig(url: LenientUri case class JdbcConfig(url: LenientUri, user: String, password: String) {
, user: String
, password: String
) {
val dbmsName: Option[String] = val dbmsName: Option[String] =
JdbcConfig.extractDbmsName(url) JdbcConfig.extractDbmsName(url)

View File

@ -22,16 +22,20 @@ trait Store[F[_]] {
object Store { object Store {
def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig def create[F[_]: Effect: ContextShift](
, connectEC: ExecutionContext jdbc: JdbcConfig,
, blocker: Blocker): Resource[F, Store[F]] = { connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, Store[F]] = {
val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass val hxa = HikariTransactor.newHikariTransactor[F](
, jdbc.url.asString jdbc.driverClass,
, jdbc.user jdbc.url.asString,
, jdbc.password jdbc.user,
, connectEC jdbc.password,
, blocker) connectEC,
blocker
)
for { for {
xa <- hxa xa <- hxa

View File

@ -21,7 +21,9 @@ trait DoobieMeta {
}) })
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces) Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
e.apply(a).noSpaces
)
implicit val metaCollectiveState: Meta[CollectiveState] = implicit val metaCollectiveState: Meta[CollectiveState] =
Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString) Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString)
@ -45,7 +47,9 @@ trait DoobieMeta {
Meta[String].imap(JobState.unsafe)(_.name) Meta[String].imap(JobState.unsafe)(_.name)
implicit val metaDirection: Meta[Direction] = implicit val metaDirection: Meta[Direction] =
Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d)) Meta[Boolean].imap(flag =>
if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction
)(d => Direction.isIncoming(d))
implicit val metaPriority: Meta[Priority] = implicit val metaPriority: Meta[Priority] =
Meta[Int].imap(Priority.fromInt)(Priority.toInt) Meta[Int].imap(Priority.fromInt)(Priority.toInt)

View File

@ -19,7 +19,9 @@ trait DoobieSyntax {
commas(fa :: fas.toList) commas(fa :: fas.toList)
def and(fs: Seq[Fragment]): Fragment = def and(fs: Seq[Fragment]): Fragment =
Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") Fragment.const(" (") ++ fs
.filter(f => !isEmpty(f))
.reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
def and(f0: Fragment, fs: Fragment*): Fragment = def and(f0: Fragment, fs: Fragment*): Fragment =
and(f0 :: fs.toList) and(f0 :: fs.toList)
@ -48,8 +50,9 @@ trait DoobieSyntax {
def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment = def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment =
Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")")) commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(
vals.map(f => sql"(" ++ f ++ sql")")
)
def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment =
selectSimple(commas(cols.map(_.f)), table, where) selectSimple(commas(cols.map(_.f)), table, where)
@ -62,7 +65,6 @@ trait DoobieSyntax {
Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++ Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++
Fragment.const(") FROM ") ++ table ++ this.where(where) Fragment.const(") FROM ") ++ table ++ this.where(where)
// def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment = // def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment =
// selectSimple(cols.map(_.prefix("a")) // selectSimple(cols.map(_.prefix("a"))
// , table ++ fr"a," ++ RCollective.table ++ fr"b" // , table ++ fr"a," ++ RCollective.table ++ fr"b"
@ -70,11 +72,12 @@ trait DoobieSyntax {
// else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b"))) // else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b")))
def selectCount(col: Column, table: Fragment, where: Fragment): Fragment = def selectCount(col: Column, table: Fragment, where: Fragment): Fragment =
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where) Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(
where
)
def deleteFrom(table: Fragment, where: Fragment): Fragment = { def deleteFrom(table: Fragment, where: Fragment): Fragment =
fr"DELETE FROM" ++ table ++ this.where(where) fr"DELETE FROM" ++ table ++ this.where(where)
}
def withCTE(ps: (String, Fragment)*): Fragment = { def withCTE(ps: (String, Fragment)*): Fragment = {
val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")") val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")")

View File

@ -1,5 +1,3 @@
package docspell.store.impl package docspell.store.impl
object Implicits extends DoobieMeta with DoobieSyntax
object Implicits extends DoobieMeta
with DoobieSyntax

View File

@ -10,7 +10,8 @@ import doobie._
import doobie.implicits._ import doobie.implicits._
final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] { final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] {
val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id)) val bitpeaceCfg =
BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
def migrate: F[Int] = def migrate: F[Int] =
FlywayMigrate.run[F](jdbc) FlywayMigrate.run[F](jdbc)
@ -24,14 +25,14 @@ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends
def bitpeace: Bitpeace[F] = def bitpeace: Bitpeace[F] =
Bitpeace(bitpeaceCfg, xa) Bitpeace(bitpeaceCfg, xa)
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = { def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] =
for { for {
save <- transact(insert).attempt save <- transact(insert).attempt
exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b))) exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b)))
} yield exist.swap match { } yield exist.swap match {
case Right(_) => AddResult.Success case Right(_) => AddResult.Success
case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.") case Left((_, true)) =>
AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((ex, _)) => AddResult.Failure(ex) case Left((ex, _)) => AddResult.Failure(ex)
} }
}
} }

View File

@ -20,11 +20,12 @@ object FlywayMigrate {
} }
logger.info(s"Using migration locations: $locations") logger.info(s"Using migration locations: $locations")
val fw = Flyway.configure(). val fw = Flyway
cleanDisabled(true). .configure()
dataSource(jdbc.url.asString, jdbc.user, jdbc.password). .cleanDisabled(true)
locations(locations: _*). .dataSource(jdbc.url.asString, jdbc.user, jdbc.password)
load() .locations(locations: _*)
.load()
fw.repair() fw.repair()
fw.migrate() fw.migrate()

View File

@ -18,7 +18,7 @@ trait ONode[F[_]] {
object ONode { object ONode {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_] : Effect](store: Store[F]): Resource[F, ONode[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, ONode[F]] =
Resource.pure(new ONode[F] { Resource.pure(new ONode[F] {
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =

View File

@ -12,40 +12,39 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
object QAttachment { object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = { def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for { for {
raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)) raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll))
n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId))) n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId)))
f <- Stream.emit(raOpt). f <- Stream
unNoneTerminate. .emit(raOpt)
map(_.fileId.id). .unNoneTerminate
flatMap(store.bitpeace.delete). .map(_.fileId.id)
compile.last .flatMap(store.bitpeace.delete)
.compile
.last
} yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0) } yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0)
}
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = { def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for { for {
n <- store.transact(RAttachment.delete(ra.id)) n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emit(ra.fileId.id). f <- Stream.emit(ra.fileId.id).flatMap(store.bitpeace.delete).compile.last
flatMap(store.bitpeace.delete).
compile.last
} yield n + f.map(_ => 1).getOrElse(0) } yield n + f.map(_ => 1).getOrElse(0)
}
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = { def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
for { for {
ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll)) ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll))
ns <- ras.traverse(deleteAttachment[F](store)) ns <- ras.traverse(deleteAttachment[F](store))
} yield ns.sum } yield ns.sum
}
def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = { def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = {
val AC = RAttachment.Columns val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns val MC = RAttachmentMeta.Columns
val IC = RItem.Columns val IC = RItem.Columns
val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ val q = fr"SELECT" ++ MC.proposals
.prefix("m")
.f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll)) fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll))
@ -55,15 +54,22 @@ object QAttachment {
} yield MetaProposalList.flatten(ml) } yield MetaProposalList.flatten(ml)
} }
def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = { def getAttachmentMeta(
attachId: Ident,
collective: Ident
): ConnectionIO[Option[RAttachmentMeta]] = {
val AC = RAttachment.Columns val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns val MC = RAttachmentMeta.Columns
val IC = RItem.Columns val IC = RItem.Columns
val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ .prefix("i")
fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective) .is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id
.prefix("a")
.is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a").is(attachId), IC.cid.prefix("i").is(collective))
q.query[RAttachmentMeta].option q.query[RAttachmentMeta].option
} }

View File

@ -8,27 +8,35 @@ import docspell.store.records.{RAttachment, RItem, RTag, RTagItem}
object QCollective { object QCollective {
case class InsightData( incoming: Int case class InsightData(incoming: Int, outgoing: Int, bytes: Long, tags: Map[String, Int])
, outgoing: Int
, bytes: Long
, tags: Map[String, Int])
def getInsights(coll: Ident): ConnectionIO[InsightData] = { def getInsights(coll: Ident): ConnectionIO[InsightData] = {
val IC = RItem.Columns val IC = RItem.Columns
val AC = RAttachment.Columns val AC = RAttachment.Columns
val TC = RTag.Columns val TC = RTag.Columns
val RC = RTagItem.Columns val RC = RTagItem.Columns
val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)). val q0 = selectCount(
query[Int].unique IC.id,
val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)). RItem.table,
query[Int].unique and(IC.cid.is(coll), IC.incoming.is(Direction.incoming))
).query[Int].unique
val q1 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique
val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++ val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++ fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++
fr"WHERE" ++ IC.cid.is(coll) fr"WHERE" ++ IC.cid.is(coll)
val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++ val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
) ++
fr"FROM" ++ RTagItem.table ++ fr"r" ++ fr"FROM" ++ RTagItem.table ++ fr"r" ++
fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++ fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++
fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++

View File

@ -13,16 +13,18 @@ import docspell.store.impl.Implicits._
import org.log4s._ import org.log4s._
object QItem { object QItem {
private [this] val logger = getLogger private[this] val logger = getLogger
case class ItemData( item: RItem case class ItemData(
, corrOrg: Option[ROrganization] item: RItem,
, corrPerson: Option[RPerson] corrOrg: Option[ROrganization],
, concPerson: Option[RPerson] corrPerson: Option[RPerson],
, concEquip: Option[REquipment] concPerson: Option[RPerson],
, inReplyTo: Option[IdRef] concEquip: Option[REquipment],
, tags: Vector[RTag] inReplyTo: Option[IdRef],
, attachments: Vector[(RAttachment, FileMeta)]) { tags: Vector[RTag],
attachments: Vector[(RAttachment, FileMeta)]
) {
def filterCollective(coll: Ident): Option[ItemData] = def filterCollective(coll: Ident): Option[ItemData] =
if (item.cid == coll) Some(this) else None if (item.cid == coll) Some(this) else None
@ -37,14 +39,35 @@ object QItem {
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++ val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg.prefix("i").is(ROrganization.Columns.oid.prefix("o")) ++ fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson.prefix("i").is(RPerson.Columns.pid.prefix("p0")) ++ .prefix("i")
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson.prefix("i").is(RPerson.Columns.pid.prefix("p1")) ++ .is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment.prefix("i").is(REquipment.Columns.eid.prefix("e")) ++ fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo.prefix("i").is(RItem.Columns.id.prefix("ref")) ++ .prefix("i")
.is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.prefix("i")
.is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.prefix("i")
.is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val q = cq.query[(RItem, Option[ROrganization], Option[RPerson], Option[RPerson], Option[REquipment], Option[IdRef])].option val q = cq
.query[
(
RItem,
Option[ROrganization],
Option[RPerson],
Option[RPerson],
Option[REquipment],
Option[IdRef]
)
]
.option
val attachs = RAttachment.findByItemWithMeta(id) val attachs = RAttachment.findByItemWithMeta(id)
val tags = RTag.findByItem(id) val tags = RTag.findByItem(id)
@ -56,35 +79,38 @@ object QItem {
} yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att)) } yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att))
} }
case class ListItem(
id: Ident,
name: String,
state: ItemState,
date: Timestamp,
dueDate: Option[Timestamp],
source: String,
direction: Direction,
created: Timestamp,
fileCount: Int,
corrOrg: Option[IdRef],
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef]
)
case class ListItem( id: Ident case class Query(
, name: String collective: Ident,
, state: ItemState name: Option[String],
, date: Timestamp states: Seq[ItemState],
, dueDate: Option[Timestamp] direction: Option[Direction],
, source: String corrPerson: Option[Ident],
, direction: Direction corrOrg: Option[Ident],
, created: Timestamp concPerson: Option[Ident],
, fileCount: Int concEquip: Option[Ident],
, corrOrg: Option[IdRef] tagsInclude: List[Ident],
, corrPerson: Option[IdRef] tagsExclude: List[Ident],
, concPerson: Option[IdRef] dateFrom: Option[Timestamp],
, concEquip: Option[IdRef]) dateTo: Option[Timestamp],
dueDateFrom: Option[Timestamp],
case class Query( collective: Ident dueDateTo: Option[Timestamp]
, name: Option[String] )
, states: Seq[ItemState]
, direction: Option[Direction]
, corrPerson: Option[Ident]
, corrOrg: Option[Ident]
, concPerson: Option[Ident]
, concEquip: Option[Ident]
, tagsInclude: List[Ident]
, tagsExclude: List[Ident]
, dateFrom: Option[Timestamp]
, dateTo: Option[Timestamp]
, dueDateFrom: Option[Timestamp]
, dueDateTo: Option[Timestamp])
def findItems(q: Query): Stream[ConnectionIO, ListItem] = { def findItems(q: Query): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns val IC = RItem.Columns
@ -97,70 +123,93 @@ object QItem {
val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name) val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name)
val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name) val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name)
val finalCols = commas(IC.id.prefix("i").f val finalCols = commas(
, IC.name.prefix("i").f IC.id.prefix("i").f,
, IC.state.prefix("i").f IC.name.prefix("i").f,
, coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) IC.state.prefix("i").f,
, IC.dueDate.prefix("i").f coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f),
, IC.source.prefix("i").f IC.dueDate.prefix("i").f,
, IC.incoming.prefix("i").f IC.source.prefix("i").f,
, IC.created.prefix("i").f IC.incoming.prefix("i").f,
, fr"COALESCE(a.num, 0)" IC.created.prefix("i").f,
, OC.oid.prefix("o0").f fr"COALESCE(a.num, 0)",
, OC.name.prefix("o0").f OC.oid.prefix("o0").f,
, PC.pid.prefix("p0").f OC.name.prefix("o0").f,
, PC.name.prefix("p0").f PC.pid.prefix("p0").f,
, PC.pid.prefix("p1").f PC.name.prefix("p0").f,
, PC.name.prefix("p1").f PC.pid.prefix("p1").f,
, EC.eid.prefix("e1").f PC.name.prefix("p1").f,
, EC.name.prefix("e1").f EC.eid.prefix("e1").f,
EC.name.prefix("e1").f
) )
val withItem = selectSimple(itemCols, RItem.table, IC.cid is q.collective) val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.collective))
val withPerson = selectSimple(personCols, RPerson.table, PC.cid is q.collective) val withPerson = selectSimple(personCols, RPerson.table, PC.cid.is(q.collective))
val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid is q.collective) val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid.is(q.collective))
val withEquips = selectSimple(equipCols, REquipment.table, EC.cid is q.collective) val withEquips = selectSimple(equipCols, REquipment.table, EC.cid.is(q.collective))
val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++
fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")"
val query = withCTE("items" -> withItem val query = withCTE(
, "persons" -> withPerson "items" -> withItem,
, "orgs" -> withOrgs "persons" -> withPerson,
, "equips" -> withEquips "orgs" -> withOrgs,
, "attachs" -> withAttach) ++ "equips" -> withEquips,
"attachs" -> withAttach
) ++
fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++ fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++
fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++ fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++ .prefix("i")
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++ .is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1")) // i.concequipment = e1.eid" fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg
.prefix("i")
.is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson
.prefix("i")
.is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
.prefix("i")
.is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
// inclusive tags are AND-ed // inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude.map(tid => val tagSelectsIncl = q.tagsInclude
selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId is tid)). .map(tid =>
map(f => sql"(" ++ f ++ sql") ") selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId.is(tid))
)
.map(f => sql"(" ++ f ++ sql") ")
// exclusive tags are OR-ed // exclusive tags are OR-ed
val tagSelectsExcl = val tagSelectsExcl =
if (q.tagsExclude.isEmpty) Fragment.empty if (q.tagsExclude.isEmpty) Fragment.empty
else selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId isOneOf q.tagsExclude) else
selectSimple(
List(RTagItem.Columns.itemId),
RTagItem.table,
RTagItem.Columns.tagId.isOneOf(q.tagsExclude)
)
val name = q.name.map(queryWildcard) val name = q.name.map(queryWildcard)
val cond = and( val cond = and(
IC.cid.prefix("i") is q.collective, IC.cid.prefix("i").is(q.collective),
IC.state.prefix("i") isOneOf q.states, IC.state.prefix("i").isOneOf(q.states),
IC.incoming.prefix("i") isOrDiscard q.direction, IC.incoming.prefix("i").isOrDiscard(q.direction),
name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty), name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty),
RPerson.Columns.pid.prefix("p0") isOrDiscard q.corrPerson, RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson),
ROrganization.Columns.oid.prefix("o0") isOrDiscard q.corrOrg, ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg),
RPerson.Columns.pid.prefix("p1") isOrDiscard q.concPerson, RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson),
REquipment.Columns.eid.prefix("e1") isOrDiscard q.concEquip, REquipment.Columns.eid.prefix("e1").isOrDiscard(q.concEquip),
if (q.tagsInclude.isEmpty) Fragment.empty if (q.tagsInclude.isEmpty) Fragment.empty
else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", else
IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
if (q.tagsExclude.isEmpty) Fragment.empty if (q.tagsExclude.isEmpty) Fragment.empty
else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")", else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")",
q.dateFrom.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d").getOrElse(Fragment.empty), q.dateFrom
q.dateTo.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d").getOrElse(Fragment.empty), .map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d")
.getOrElse(Fragment.empty),
q.dateTo
.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d")
.getOrElse(Fragment.empty),
q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty), q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty),
q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty) q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty)
) )
@ -171,7 +220,6 @@ object QItem {
frag.query[ListItem].stream frag.query[ListItem].stream
} }
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
for { for {
tn <- store.transact(RTagItem.deleteItemTags(itemId)) tn <- store.transact(RTagItem.deleteItemTags(itemId))
@ -183,7 +231,9 @@ object QItem {
val IC = RItem.Columns val IC = RItem.Columns
val AC = RAttachment.Columns val AC = RAttachment.Columns
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc) fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
q.query[RItem].to[Vector] q.query[RItem].to[Vector]

View File

@ -13,29 +13,38 @@ import fs2.Stream
import org.log4s._ import org.log4s._
object QJob { object QJob {
private [this] val logger = getLogger private[this] val logger = getLogger
def takeNextJob[F[_]: Effect](store: Store[F])(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = { def takeNextJob[F[_]: Effect](
Stream.range(0, 10). store: Store[F]
evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)). )(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] =
evalTap({ x => Stream
if (x.isLeft) logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.") .range(0, 10)
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
.evalTap({ x =>
if (x.isLeft)
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
else ().pure[F] else ().pure[F]
}). })
find(_.isRight). .find(_.isRight)
flatMap({ .flatMap({
case Right(job) => case Right(job) =>
Stream.emit(job) Stream.emit(job)
case Left(_) => case Left(_) =>
Stream.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")).map(_ => None) Stream
}). .eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
compile.last.map(_.flatten) .map(_ => None)
} })
.compile
.last
.map(_.flatten)
private def takeNextJob1[F[_]: Effect](store: Store[F])( priority: Ident => F[Priority] private def takeNextJob1[F[_]: Effect](store: Store[F])(
, worker: Ident priority: Ident => F[Priority],
, retryPause: Duration worker: Ident,
, currentTry: Int): F[Either[Unit, Option[RJob]]] = { retryPause: Duration,
currentTry: Int
): F[Either[Unit, Option[RJob]]] = {
//if this fails, we have to restart takeNextJob //if this fails, we have to restart takeNextJob
def markJob(job: RJob): F[Either[Unit, RJob]] = def markJob(job: RJob): F[Either[Unit, RJob]] =
store.transact(for { store.transact(for {
@ -51,7 +60,9 @@ object QJob {
_ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}") _ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}")
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F]) prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
_ <- logger.ftrace[F](s"Looking for job of prio $prio") _ <- logger.ftrace[F](s"Looking for job of prio $prio")
job <- group.map(g => store.transact(selectNextJob(g, prio, retryPause, now))).getOrElse((None: Option[RJob]).pure[F]) job <- group
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F])
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}") _ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
res <- job.traverse(j => markJob(j)) res <- job.traverse(j => markJob(j))
} yield res.map(_.map(_.some)).getOrElse { } yield res.map(_.map(_.some)).getOrElse {
@ -60,7 +71,11 @@ object QJob {
} }
} }
def selectNextGroup(worker: Ident, now: Timestamp, initialPause: Duration): ConnectionIO[Option[Ident]] = { def selectNextGroup(
worker: Ident,
now: Timestamp,
initialPause: Duration
): ConnectionIO[Option[Ident]] = {
val JC = RJob.Columns val JC = RJob.Columns
val waiting: JobState = JobState.Waiting val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck val stuck: JobState = JobState.Stuck
@ -72,21 +87,30 @@ object QJob {
val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++ val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++
fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}" fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}"
val stateCond = or(jstate is waiting, and(jstate is stuck, stuckTrigger ++ fr"< ${now.toMillis}")) val stateCond =
or(jstate.is(waiting), and(jstate.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++ val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++
fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++ fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++
fr"WHERE" ++ and(uworker is worker, stateCond) ++ fr"WHERE" ++ and(uworker.is(worker), stateCond) ++
fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++ val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
fr"WHERE" ++ stateCond fr"WHERE" ++ stateCond
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null" val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
union.query[Ident].to[List].map(_.headOption) // either one or two results, but may be empty if RJob table is empty union
.query[Ident]
.to[List]
.map(_.headOption) // either one or two results, but may be empty if RJob table is empty
} }
def selectNextJob(group: Ident, prio: Priority, initialPause: Duration, now: Timestamp): ConnectionIO[Option[RJob]] = { def selectNextJob(
group: Ident,
prio: Priority,
initialPause: Duration,
now: Timestamp
): ConnectionIO[Option[RJob]] = {
val JC = RJob.Columns val JC = RJob.Columns
val psort = val psort =
if (prio == Priority.High) JC.priority.desc if (prio == Priority.High) JC.priority.desc
@ -94,9 +118,17 @@ object QJob {
val waiting: JobState = JobState.Waiting val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck val stuck: JobState = JobState.Stuck
val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries) ++ fr"* ${initialPause.millis}" val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(
val sql = selectSimple(JC.all, RJob.table, JC.retries
and(JC.group is group, or(JC.state is waiting, and(JC.state is stuck, stuckTrigger ++ fr"< ${now.toMillis}")))) ++ ) ++ fr"* ${initialPause.millis}"
val sql = selectSimple(
JC.all,
RJob.table,
and(
JC.group.is(group),
or(JC.state.is(waiting), and(JC.state.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
)
) ++
orderBy(JC.state.asc, psort, JC.submitted.asc) ++ orderBy(JC.state.asc, psort, JC.submitted.asc) ++
fr"LIMIT 1" fr"LIMIT 1"
@ -150,9 +182,8 @@ object QJob {
def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] = def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] =
store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max)) store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max))
def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = { def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] =
store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ()) store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ())
}
def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] = def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] =
store.transact(RJob.findFromIds(ids)) store.transact(RJob.findFromIds(ids))
@ -165,10 +196,17 @@ object QJob {
def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = { def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = {
val refDate = now.minusHours(24) val refDate = now.minusHours(24)
val sql = selectSimple(JC.all, RJob.table, val sql = selectSimple(
and(JC.group is collective, JC.all,
or(and(JC.state.isOneOf(done.toSeq), JC.submitted isGt refDate) RJob.table,
, JC.state.isOneOf((running ++ waiting).toSeq)))) and(
JC.group.is(collective),
or(
and(JC.state.isOneOf(done.toSeq), JC.submitted.isGt(refDate)),
JC.state.isOneOf((running ++ waiting).toSeq)
)
)
)
(sql ++ orderBy(JC.submitted.desc)).query[RJob].stream (sql ++ orderBy(JC.submitted.desc)).query[RJob].stream
} }

View File

@ -12,10 +12,12 @@ import org.log4s._
object QLogin { object QLogin {
private[this] val logger = getLogger private[this] val logger = getLogger
case class Data( account: AccountId case class Data(
, password: Password account: AccountId,
, collectiveState: CollectiveState password: Password,
, userState: UserState) collectiveState: CollectiveState,
userState: UserState
)
def findUser(acc: AccountId): ConnectionIO[Option[Data]] = { def findUser(acc: AccountId): ConnectionIO[Option[Data]] = {
val ucid = UC.cid.prefix("u") val ucid = UC.cid.prefix("u")
@ -26,9 +28,10 @@ object QLogin {
val ccid = CC.id.prefix("c") val ccid = CC.id.prefix("c")
val sql = selectSimple( val sql = selectSimple(
List(ucid,login,pass,cstate,ustate), List(ucid, login, pass, cstate, ustate),
RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c", RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c",
and(ucid is ccid, login is acc.user, ucid is acc.collective)) and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective))
)
logger.trace(s"SQL : $sql") logger.trace(s"SQL : $sql")
sql.query[Data].option sql.query[Data].option

View File

@ -12,16 +12,24 @@ import docspell.store.records._
object QOrganization { object QOrganization {
def findOrgAndContact(coll: Ident, order: OC.type => Column): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = { def findOrgAndContact(
ROrganization.findAll(coll, order). coll: Ident,
evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs))) order: OC.type => Column
} ): Stream[ConnectionIO, (ROrganization, Vector[RContact])] =
def findPersonAndContact(coll: Ident, order: PC.type => Column): Stream[ConnectionIO, (RPerson, Vector[RContact])] = { ROrganization
RPerson.findAll(coll, order). .findAll(coll, order)
evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs))) .evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs)))
} def findPersonAndContact(
coll: Ident,
order: PC.type => Column
): Stream[ConnectionIO, (RPerson, Vector[RContact])] =
RPerson.findAll(coll, order).evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs)))
def addOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { def addOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for { val insert = for {
n <- ROrganization.insert(org) n <- ROrganization.insert(org)
cs <- contacts.toList.traverse(RContact.insert) cs <- contacts.toList.traverse(RContact.insert)
@ -32,7 +40,11 @@ object QOrganization {
store => store.add(insert, exists) store => store.add(insert, exists)
} }
def addPerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { def addPerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for { val insert = for {
n <- RPerson.insert(person) n <- RPerson.insert(person)
cs <- contacts.toList.traverse(RContact.insert) cs <- contacts.toList.traverse(RContact.insert)
@ -43,7 +55,11 @@ object QOrganization {
store => store.add(insert, exists) store => store.add(insert, exists)
} }
def updateOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { def updateOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for { val insert = for {
n <- ROrganization.update(org) n <- ROrganization.update(org)
d <- RContact.deleteOrg(org.oid) d <- RContact.deleteOrg(org.oid)
@ -55,7 +71,11 @@ object QOrganization {
store => store.add(insert, exists) store => store.add(insert, exists)
} }
def updatePerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { def updatePerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for { val insert = for {
n <- RPerson.update(person) n <- RPerson.update(person)
d <- RContact.deletePerson(person.pid) d <- RContact.deletePerson(person.pid)
@ -67,20 +87,18 @@ object QOrganization {
store => store.add(insert, exists) store => store.add(insert, exists)
} }
def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = { def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] =
for { for {
n0 <- RItem.removeCorrOrg(collective, orgId) n0 <- RItem.removeCorrOrg(collective, orgId)
n1 <- RContact.deleteOrg(orgId) n1 <- RContact.deleteOrg(orgId)
n2 <- ROrganization.delete(orgId, collective) n2 <- ROrganization.delete(orgId, collective)
} yield n0 + n1 + n2 } yield n0 + n1 + n2
}
def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = { def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] =
for { for {
n0 <- RItem.removeCorrPerson(collective, personId) n0 <- RItem.removeCorrPerson(collective, personId)
n1 <- RItem.removeConcPerson(collective, personId) n1 <- RItem.removeConcPerson(collective, personId)
n2 <- RContact.deletePerson(personId) n2 <- RContact.deletePerson(personId)
n3 <- RPerson.delete(personId, collective) n3 <- RPerson.delete(personId, collective)
} yield n0 + n1 + n2 + n3 } yield n0 + n1 + n2 + n3
}
} }

View File

@ -21,22 +21,29 @@ trait JobQueue[F[_]] {
object JobQueue { object JobQueue {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_] : Effect](store: Store[F]): Resource[F, JobQueue[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, JobQueue[F]] =
Resource.pure(new JobQueue[F] { Resource.pure(new JobQueue[F] {
def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = def nextJob(
prio: Ident => F[Priority],
worker: Ident,
retryPause: Duration
): F[Option[RJob]] =
logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause) logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
def insert(job: RJob): F[Unit] = def insert(job: RJob): F[Unit] =
store.transact(RJob.insert(job)). store
flatMap({ n => .transact(RJob.insert(job))
if (n != 1) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n")) .flatMap({ n =>
if (n != 1)
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F] else ().pure[F]
}) })
def insertAll(jobs: Seq[RJob]): F[Unit] = def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList.traverse(j => insert(j).attempt). jobs.toList
map(_.foreach { .traverse(j => insert(j).attempt)
.map(_.foreach {
case Right(()) => case Right(()) =>
case Left(ex) => case Left(ex) =>
logger.error(ex)("Could not insert job. Skipping it.") logger.error(ex)("Could not insert job. Skipping it.")

View File

@ -7,14 +7,14 @@ import docspell.common._
import docspell.store.impl._ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class RAttachment( id: Ident case class RAttachment(
, itemId: Ident id: Ident,
, fileId: Ident itemId: Ident,
, position: Int fileId: Ident,
, created: Timestamp position: Int,
, name: Option[String]) { created: Timestamp,
name: Option[String]
} ) {}
object RAttachment { object RAttachment {
@ -32,25 +32,34 @@ object RAttachment {
import Columns._ import Columns._
def insert(v: RAttachment): ConnectionIO[Int] = def insert(v: RAttachment): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}").update.run insertRow(
table,
all,
fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}"
).update.run
def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] = def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(all, table, id is attachId).query[RAttachment].option selectSimple(all, table, id.is(attachId)).query[RAttachment].option
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = { def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(all.map(_.prefix("a")), table ++ fr"a," ++ RItem.table ++ fr"i", and( selectSimple(
all.map(_.prefix("a")),
table ++ fr"a," ++ RItem.table ++ fr"i",
and(
fr"a.itemid = i.itemid", fr"a.itemid = i.itemid",
id.prefix("a") is attachId, id.prefix("a").is(attachId),
RItem.Columns.cid.prefix("i") is collective RItem.Columns.cid.prefix("i").is(collective)
)).query[RAttachment].option )
} ).query[RAttachment].option
def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] = def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] =
selectSimple(all, table, itemId is id).query[RAttachment].to[Vector] selectSimple(all, table, itemId.is(id)).query[RAttachment].to[Vector]
def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = { def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = {
val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id.prefix("i").is(itemId.prefix("a")) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id
.prefix("i")
.is(itemId.prefix("a")) ++
fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll))
q.query[RAttachment].to[Vector] q.query[RAttachment].to[Vector]
} }
@ -65,7 +74,7 @@ object RAttachment {
def delete(attachId: Ident): ConnectionIO[Int] = def delete(attachId: Ident): ConnectionIO[Int] =
for { for {
n0 <- RAttachmentMeta.delete(attachId) n0 <- RAttachmentMeta.delete(attachId)
n1 <- deleteFrom(table, id is attachId).update.run n1 <- deleteFrom(table, id.is(attachId)).update.run
} yield n0 + n1 } yield n0 + n1
} }

View File

@ -7,12 +7,12 @@ import docspell.common._
import docspell.store.impl._ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class RAttachmentMeta(id: Ident case class RAttachmentMeta(
, content: Option[String] id: Ident,
, nerlabels: List[NerLabel] content: Option[String],
, proposals: MetaProposalList) { nerlabels: List[NerLabel],
proposals: MetaProposalList
} ) {}
object RAttachmentMeta { object RAttachmentMeta {
def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty) def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty)
@ -32,7 +32,7 @@ object RAttachmentMeta {
insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run
def exists(attachId: Ident): ConnectionIO[Boolean] = def exists(attachId: Ident): ConnectionIO[Boolean] =
selectCount(id, table, id is attachId).query[Int].unique.map(_ > 0) selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0)
def upsert(v: RAttachmentMeta): ConnectionIO[Int] = def upsert(v: RAttachmentMeta): ConnectionIO[Int] =
for { for {
@ -41,22 +41,34 @@ object RAttachmentMeta {
} yield n1 } yield n1
def update(v: RAttachmentMeta): ConnectionIO[Int] = def update(v: RAttachmentMeta): ConnectionIO[Int] =
updateRow(table, id is v.id, commas( updateRow(
content setTo v.content, table,
nerlabels setTo v.nerlabels, id.is(v.id),
proposals setTo v.proposals commas(
)).update.run content.setTo(v.content),
nerlabels.setTo(v.nerlabels),
proposals.setTo(v.proposals)
)
).update.run
def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] = def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] =
updateRow(table, id is mid, commas( updateRow(
nerlabels setTo labels table,
)).update.run id.is(mid),
commas(
nerlabels.setTo(labels)
)
).update.run
def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] = def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] =
updateRow(table, id is mid, commas( updateRow(
proposals setTo plist table,
)).update.run id.is(mid),
commas(
proposals.setTo(plist)
)
).update.run
def delete(attachId: Ident): ConnectionIO[Int] = def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id is attachId).update.run deleteFrom(table, id.is(attachId)).update.run
} }

View File

@ -7,10 +7,7 @@ import doobie._
import doobie.implicits._ import doobie.implicits._
import fs2.Stream import fs2.Stream
case class RCollective( id: Ident case class RCollective(id: Ident, state: CollectiveState, language: Language, created: Timestamp)
, state: CollectiveState
, language: Language
, created: Timestamp)
object RCollective { object RCollective {
@ -29,30 +26,38 @@ object RCollective {
import Columns._ import Columns._
def insert(value: RCollective): ConnectionIO[Int] = { def insert(value: RCollective): ConnectionIO[Int] = {
val sql = insertRow(table, Columns.all, fr"${value.id},${value.state},${value.language},${value.created}") val sql = insertRow(
table,
Columns.all,
fr"${value.id},${value.state},${value.language},${value.created}"
)
sql.update.run sql.update.run
} }
def update(value: RCollective): ConnectionIO[Int] = { def update(value: RCollective): ConnectionIO[Int] = {
val sql = updateRow(table, id is value.id, commas( val sql = updateRow(
state setTo value.state table,
)) id.is(value.id),
commas(
state.setTo(value.state)
)
)
sql.update.run sql.update.run
} }
def findLanguage(cid: Ident): ConnectionIO[Option[Language]] = def findLanguage(cid: Ident): ConnectionIO[Option[Language]] =
selectSimple(List(language), table, id is cid).query[Option[Language]].unique selectSimple(List(language), table, id.is(cid)).query[Option[Language]].unique
def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] = def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] =
updateRow(table, id is cid, language setTo lang).update.run updateRow(table, id.is(cid), language.setTo(lang)).update.run
def findById(cid: Ident): ConnectionIO[Option[RCollective]] = { def findById(cid: Ident): ConnectionIO[Option[RCollective]] = {
val sql = selectSimple(all, table, id is cid) val sql = selectSimple(all, table, id.is(cid))
sql.query[RCollective].option sql.query[RCollective].option
} }
def existsById(cid: Ident): ConnectionIO[Boolean] = { def existsById(cid: Ident): ConnectionIO[Boolean] = {
val sql = selectCount(id, table, id is cid) val sql = selectCount(id, table, id.is(cid))
sql.query[Int].unique.map(_ > 0) sql.query[Int].unique.map(_ > 0)
} }

View File

@ -6,14 +6,13 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class RContact( case class RContact(
contactId: Ident contactId: Ident,
, value: String value: String,
, kind: ContactKind kind: ContactKind,
, personId: Option[Ident] personId: Option[Ident],
, orgId: Option[Ident] orgId: Option[Ident],
, created: Timestamp) { created: Timestamp
) {}
}
object RContact { object RContact {
@ -26,48 +25,55 @@ object RContact {
val personId = Column("pid") val personId = Column("pid")
val orgId = Column("oid") val orgId = Column("oid")
val created = Column("created") val created = Column("created")
val all = List(contactId, value,kind, personId, orgId, created) val all = List(contactId, value, kind, personId, orgId, created)
} }
import Columns._ import Columns._
def insert(v: RContact): ConnectionIO[Int] = { def insert(v: RContact): ConnectionIO[Int] = {
val sql = insertRow(table, all, val sql = insertRow(
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}") table,
all,
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}"
)
sql.update.run sql.update.run
} }
def update(v: RContact): ConnectionIO[Int] = { def update(v: RContact): ConnectionIO[Int] = {
val sql = updateRow(table, contactId is v.contactId, commas( val sql = updateRow(
value setTo v.value, table,
kind setTo v.kind, contactId.is(v.contactId),
personId setTo v.personId, commas(
orgId setTo v.orgId value.setTo(v.value),
)) kind.setTo(v.kind),
personId.setTo(v.personId),
orgId.setTo(v.orgId)
)
)
sql.update.run sql.update.run
} }
def delete(v: RContact): ConnectionIO[Int] = def delete(v: RContact): ConnectionIO[Int] =
deleteFrom(table, contactId is v.contactId).update.run deleteFrom(table, contactId.is(v.contactId)).update.run
def deleteOrg(oid: Ident): ConnectionIO[Int] = def deleteOrg(oid: Ident): ConnectionIO[Int] =
deleteFrom(table, orgId is oid).update.run deleteFrom(table, orgId.is(oid)).update.run
def deletePerson(pid: Ident): ConnectionIO[Int] = def deletePerson(pid: Ident): ConnectionIO[Int] =
deleteFrom(table, personId is pid).update.run deleteFrom(table, personId.is(pid)).update.run
def findById(id: Ident): ConnectionIO[Option[RContact]] = { def findById(id: Ident): ConnectionIO[Option[RContact]] = {
val sql = selectSimple(all, table, contactId is id) val sql = selectSimple(all, table, contactId.is(id))
sql.query[RContact].option sql.query[RContact].option
} }
def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = { def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, personId is pid) val sql = selectSimple(all, table, personId.is(pid))
sql.query[RContact].to[Vector] sql.query[RContact].to[Vector]
} }
def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = { def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, orgId is oid) val sql = selectSimple(all, table, orgId.is(oid))
sql.query[RContact].to[Vector] sql.query[RContact].to[Vector]
} }
} }

View File

@ -5,13 +5,7 @@ import docspell.common._
import docspell.store.impl._ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class REquipment( case class REquipment(eid: Ident, cid: Ident, name: String, created: Timestamp) {}
eid: Ident
, cid: Ident
, name: String
, created: Timestamp) {
}
object REquipment { object REquipment {
@ -22,44 +16,47 @@ object REquipment {
val cid = Column("cid") val cid = Column("cid")
val name = Column("name") val name = Column("name")
val created = Column("created") val created = Column("created")
val all = List(eid,cid,name,created) val all = List(eid, cid, name, created)
} }
import Columns._ import Columns._
def insert(v: REquipment): ConnectionIO[Int] = { def insert(v: REquipment): ConnectionIO[Int] = {
val sql = insertRow(table, all, val sql = insertRow(table, all, fr"${v.eid},${v.cid},${v.name},${v.created}")
fr"${v.eid},${v.cid},${v.name},${v.created}")
sql.update.run sql.update.run
} }
def update(v: REquipment): ConnectionIO[Int] = { def update(v: REquipment): ConnectionIO[Int] = {
val sql = updateRow(table, and(eid is v.eid, cid is v.cid), commas( val sql = updateRow(
cid setTo v.cid, table,
name setTo v.name and(eid.is(v.eid), cid.is(v.cid)),
)) commas(
cid.setTo(v.cid),
name.setTo(v.name)
)
)
sql.update.run sql.update.run
} }
def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = { def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = {
val sql = selectCount(eid, table, and(cid is coll, name is ename)) val sql = selectCount(eid, table, and(cid.is(coll), name.is(ename)))
sql.query[Int].unique.map(_ > 0) sql.query[Int].unique.map(_ > 0)
} }
def findById(id: Ident): ConnectionIO[Option[REquipment]] = { def findById(id: Ident): ConnectionIO[Option[REquipment]] = {
val sql = selectSimple(all, table, eid is id) val sql = selectSimple(all, table, eid.is(id))
sql.query[REquipment].option sql.query[REquipment].option
} }
def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = { def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[REquipment].to[Vector] sql.query[REquipment].to[Vector]
} }
def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] =
selectSimple(List(eid, name), table, and(cid is coll, selectSimple(List(eid, name), table, and(cid.is(coll), name.lowerLike(equipName)))
name.lowerLike(equipName))). .query[IdRef]
query[IdRef].to[Vector] .to[Vector]
def delete(id: Ident, coll: Ident): ConnectionIO[Int] = def delete(id: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(eid is id, cid is coll)).update.run deleteFrom(table, and(eid.is(id), cid.is(coll))).update.run
} }

View File

@ -8,9 +8,7 @@ import docspell.common._
import docspell.store.impl._ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class RInvitation(id: Ident, created: Timestamp) { case class RInvitation(id: Ident, created: Timestamp) {}
}
object RInvitation { object RInvitation {
@ -33,18 +31,16 @@ object RInvitation {
insertRow(table, all, fr"${v.id},${v.created}").update.run insertRow(table, all, fr"${v.id},${v.created}").update.run
def insertNew: ConnectionIO[RInvitation] = def insertNew: ConnectionIO[RInvitation] =
generate[ConnectionIO]. generate[ConnectionIO].flatMap(v => insert(v).map(_ => v))
flatMap(v => insert(v).map(_ => v))
def findById(invite: Ident): ConnectionIO[Option[RInvitation]] = def findById(invite: Ident): ConnectionIO[Option[RInvitation]] =
selectSimple(all, table, id is invite).query[RInvitation].option selectSimple(all, table, id.is(invite)).query[RInvitation].option
def delete(invite: Ident): ConnectionIO[Int] = def delete(invite: Ident): ConnectionIO[Int] =
deleteFrom(table, id is invite).update.run deleteFrom(table, id.is(invite)).update.run
def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = { def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = {
val get = selectCount(id, table, and(id is invite, created isGt minCreated)). val get = selectCount(id, table, and(id.is(invite), created.isGt(minCreated))).query[Int].unique
query[Int].unique
for { for {
inv <- get inv <- get
_ <- delete(invite) _ <- delete(invite)

View File

@ -8,32 +8,55 @@ import docspell.common._
import docspell.store.impl._ import docspell.store.impl._
import docspell.store.impl.Implicits._ import docspell.store.impl.Implicits._
case class RItem( id: Ident case class RItem(
, cid: Ident id: Ident,
, name: String cid: Ident,
, itemDate: Option[Timestamp] name: String,
, source: String itemDate: Option[Timestamp],
, direction: Direction source: String,
, state: ItemState direction: Direction,
, corrOrg: Option[Ident] state: ItemState,
, corrPerson: Option[Ident] corrOrg: Option[Ident],
, concPerson: Option[Ident] corrPerson: Option[Ident],
, concEquipment: Option[Ident] concPerson: Option[Ident],
, inReplyTo: Option[Ident] concEquipment: Option[Ident],
, dueDate: Option[Timestamp] inReplyTo: Option[Ident],
, created: Timestamp dueDate: Option[Timestamp],
, updated: Timestamp created: Timestamp,
, notes: Option[String]) { updated: Timestamp,
notes: Option[String]
} ) {}
object RItem { object RItem {
def newItem[F[_]: Sync](cid: Ident, name: String, source: String, direction: Direction, state: ItemState): F[RItem] = def newItem[F[_]: Sync](
cid: Ident,
name: String,
source: String,
direction: Direction,
state: ItemState
): F[RItem] =
for { for {
now <- Timestamp.current[F] now <- Timestamp.current[F]
id <- Ident.randomId[F] id <- Ident.randomId[F]
} yield RItem(id, cid, name, None, source, direction, state, None, None, None, None, None, None, now, now, None) } yield RItem(
id,
cid,
name,
None,
source,
direction,
state,
None,
None,
None,
None,
None,
None,
now,
now,
None
)
val table = fr"item" val table = fr"item"
@ -54,110 +77,189 @@ object RItem {
val created = Column("created") val created = Column("created")
val updated = Column("updated") val updated = Column("updated")
val notes = Column("notes") val notes = Column("notes")
val all = List(id, cid, name, itemDate, source, incoming, state, corrOrg, val all = List(
corrPerson, concPerson, concEquipment, inReplyTo, dueDate, created, updated, notes) id,
cid,
name,
itemDate,
source,
incoming,
state,
corrOrg,
corrPerson,
concPerson,
concEquipment,
inReplyTo,
dueDate,
created,
updated,
notes
)
} }
import Columns._ import Columns._
def insert(v: RItem): ConnectionIO[Int] = def insert(v: RItem): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++ insertRow(
table,
all,
fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++
fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++ fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++
fr"${v.created},${v.updated},${v.notes}").update.run fr"${v.created},${v.updated},${v.notes}"
).update.run
def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] = def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] =
selectSimple(List(cid), table, id is itemId).query[Ident].option selectSimple(List(cid), table, id.is(itemId)).query[Ident].option
def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] = def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, id is itemId, commas(state setTo itemState, updated setTo t)).update.run n <- updateRow(table, id.is(itemId), commas(state.setTo(itemState), updated.setTo(t))).update.run
} yield n } yield n
def updateStateForCollective(itemId: Ident, itemState: ItemState, coll: Ident): ConnectionIO[Int] = def updateStateForCollective(
itemId: Ident,
itemState: ItemState,
coll: Ident
): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(state setTo itemState, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t))
).update.run
} yield n } yield n
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] = def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(incoming setTo dir, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t))
).update.run
} yield n } yield n
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] = def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrOrg setTo org, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t))
).update.run
} yield n } yield n
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] = def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(cid is coll, corrOrg is Some(currentOrg)), commas(corrOrg setTo(None: Option[Ident]), updated setTo t)).update.run n <- updateRow(
table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n } yield n
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrPerson setTo person, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t))
).update.run
} yield n } yield n
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(cid is coll, corrPerson is Some(currentPerson)), commas(corrPerson setTo(None: Option[Ident]), updated setTo t)).update.run n <- updateRow(
table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n } yield n
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concPerson setTo person, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t))
).update.run
} yield n } yield n
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(cid is coll, concPerson is Some(currentPerson)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run n <- updateRow(
table,
and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n } yield n
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] = def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concEquipment setTo equip, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run
} yield n } yield n
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] = def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(cid is coll, concEquipment is Some(currentEquip)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run n <- updateRow(
table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n } yield n
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(notes setTo text, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t))
).update.run
} yield n } yield n
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(name setTo itemName, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t))
).update.run
} yield n } yield n
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(itemDate setTo date, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t))
).update.run
} yield n } yield n
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(dueDate setTo date, updated setTo t)).update.run n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t))
).update.run
} yield n } yield n
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] = def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(id is itemId, cid is coll)).update.run deleteFrom(table, and(id.is(itemId), cid.is(coll))).update.run
} }

Some files were not shown because too many files have changed in this diff Show More