Apply scalafmt to all files

This commit is contained in:
Eike Kettner 2019-12-30 21:44:13 +01:00
parent 57e274e2b0
commit fc3e22e399
133 changed files with 3003 additions and 2112 deletions

View File

@ -27,7 +27,11 @@ trait BackendApp[F[_]] {
object BackendApp {
def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] =
def create[F[_]: ConcurrentEffect](
cfg: Config,
store: Store[F],
httpClientEc: ExecutionContext
): Resource[F, BackendApp[F]] =
for {
queue <- JobQueue(store)
loginImpl <- Login[F](store)
@ -55,10 +59,12 @@ object BackendApp {
val item = itemImpl
}
def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config
, connectEC: ExecutionContext
, httpClientEc: ExecutionContext
, blocker: Blocker): Resource[F, BackendApp[F]] =
def apply[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, BackendApp[F]] =
for {
store <- Store.create(cfg.jdbc, connectEC, blocker)
backend <- create(cfg, store, httpClientEc)

View File

@ -4,11 +4,7 @@ import docspell.backend.signup.{Config => SignupConfig}
import docspell.common.MimeType
import docspell.store.JdbcConfig
case class Config( jdbc: JdbcConfig
, signup: SignupConfig
, files: Config.Files) {
}
case class Config(jdbc: JdbcConfig, signup: SignupConfig, files: Config.Files) {}
object Config {

View File

@ -50,14 +50,13 @@ object AuthToken {
Left("Invalid authenticator")
}
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = {
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for {
salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key)
} yield cd.copy(sig = sig)
}
private def sign(cd: AuthToken, key: ByteVector): String = {
val raw = cd.millis.toString + cd.account.asString + cd.salt

View File

@ -49,7 +49,8 @@ object Login {
def invalidTime: Result = InvalidTime
}
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] {
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] =
Resource.pure(new Login[F] {
def loginSession(config: Config)(sessionKey: String): F[Result] =
AuthToken.fromString(sessionKey) match {
@ -61,10 +62,10 @@ object Login {
Result.invalidAuth.pure[F]
}
def loginUserPass(config: Config)(up: UserPass): F[Result] = {
def loginUserPass(config: Config)(up: UserPass): F[Result] =
AccountId.parse(up.user) match {
case Right(acc) =>
val okResult=
val okResult =
store.transact(RUser.updateLogin(acc)) *>
AuthToken.user(acc, config.serverSecret).map(Result.ok)
for {
@ -76,7 +77,6 @@ object Login {
case Left(_) =>
Result.invalidAuth.pure[F]
}
}
private def check(given: String)(data: QLogin.Data): Boolean = {
val collOk = data.collectiveState == CollectiveState.Active ||

View File

@ -25,7 +25,11 @@ trait OCollective[F[_]] {
def insights(collective: Ident): F[InsightData]
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult]
def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult]
}
object OCollective {
@ -63,38 +67,46 @@ object OCollective {
}
}
def apply[F[_]:Effect](store: Store[F]): Resource[F, OCollective[F]] =
def apply[F[_]: Effect](store: Store[F]): Resource[F, OCollective[F]] =
Resource.pure(new OCollective[F] {
def find(name: Ident): F[Option[RCollective]] =
store.transact(RCollective.findById(name))
def updateLanguage(collective: Ident, lang: Language): F[AddResult] =
store.transact(RCollective.updateLanguage(collective, lang)).
attempt.map(AddResult.fromUpdate)
store
.transact(RCollective.updateLanguage(collective, lang))
.attempt
.map(AddResult.fromUpdate)
def listUser(collective: Ident): F[Vector[RUser]] = {
def listUser(collective: Ident): F[Vector[RUser]] =
store.transact(RUser.findAll(collective, _.login))
}
def add(s: RUser): F[AddResult] =
store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login))
store.add(
RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))),
RUser.exists(s.login)
)
def update(s: RUser): F[AddResult] =
store.add(RUser.update(s), RUser.exists(s.login))
def deleteUser(login: Ident, collective: Ident): F[AddResult] =
store.transact(RUser.delete(login, collective)).
attempt.map(AddResult.fromUpdate)
store.transact(RUser.delete(login, collective)).attempt.map(AddResult.fromUpdate)
def insights(collective: Ident): F[InsightData] =
store.transact(QCollective.getInsights(collective))
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = {
def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult] = {
val q = for {
optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
n <- check
.filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match {
case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed

View File

@ -17,7 +17,6 @@ trait OEquipment[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult]
}
object OEquipment {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] =
@ -46,9 +45,7 @@ object OEquipment {
n0 <- RItem.removeConcEquip(collective, id)
n1 <- REquipment.delete(id, collective)
} yield n0 + n1
store.transact(io).
attempt.
map(AddResult.fromUpdate)
store.transact(io).attempt.map(AddResult.fromUpdate)
}
})
}

View File

@ -62,90 +62,98 @@ object OItem {
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] =
Resource.pure(new OItem[F] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store.transact(QItem.findItem(id)).
map(opt => opt.flatMap(_.filterCollective(collective)))
store.transact(QItem.findItem(id)).map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = {
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] =
store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector
}
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = {
store.transact(RAttachment.findByIdAndCollective(id, collective)).
flatMap({
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last.
map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m)))))
store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
)
)
case None =>
(None: Option[AttachmentData[F]]).pure[F]
})
}
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for {
cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO]
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
} yield nd + ni
store.transact(db).
attempt.
map(AddResult.fromUpdate)
store.transact(db).attempt.map(AddResult.fromUpdate)
}
def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] =
store.transact(RItem.updateDirection(item, collective, direction)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateDirection(item, collective, direction))
.attempt
.map(AddResult.fromUpdate)
def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrOrg(item, collective, org)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateCorrOrg(item, collective, org)).attempt.map(AddResult.fromUpdate)
def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrPerson(item, collective, person)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateCorrPerson(item, collective, person))
.attempt
.map(AddResult.fromUpdate)
def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcPerson(item, collective, person)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateConcPerson(item, collective, person))
.attempt
.map(AddResult.fromUpdate)
def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcEquip(item, collective, equip)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateConcEquip(item, collective, equip))
.attempt
.map(AddResult.fromUpdate)
def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] =
store.transact(RItem.updateNotes(item, collective, notes)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateNotes(item, collective, notes)).attempt.map(AddResult.fromUpdate)
def setName(item: Ident, name: String, collective: Ident): F[AddResult] =
store.transact(RItem.updateName(item, collective, name)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateName(item, collective, name)).attempt.map(AddResult.fromUpdate)
def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] =
store.transact(RItem.updateStateForCollective(item, state, collective)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateStateForCollective(item, state, collective))
.attempt
.map(AddResult.fromUpdate)
def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDate(item, collective, date)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateDate(item, collective, date)).attempt.map(AddResult.fromUpdate)
def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDueDate(item, collective, date)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateDueDate(item, collective, date))
.attempt
.map(AddResult.fromUpdate)
def delete(itemId: Ident, collective: Ident): F[Int] =
QItem.delete(store)(itemId, collective)

View File

@ -36,15 +36,19 @@ object OJob {
jobs.filter(_.job.state == JobState.Running)
}
def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] =
def apply[F[_]: ConcurrentEffect](
store: Store[F],
clientEC: ExecutionContext
): Resource[F, OJob[F]] =
Resource.pure(new OJob[F] {
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = {
store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)).
map(t => JobDetail(t._1, t._2)).
compile.toVector.
map(CollectiveQueueState)
}
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
store
.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong))
.map(t => JobDetail(t._1, t._2))
.compile
.toVector
.map(CollectiveQueueState)
def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = {
def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] =
@ -66,8 +70,9 @@ object OJob {
}
def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] =
OJoex.cancelJob(job.id, worker, store, clientEC).
map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
OJoex
.cancelJob(job.id, worker, store, clientEC)
.map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
for {
tryDel <- store.transact(tryDelete)

View File

@ -13,24 +13,32 @@ import scala.concurrent.ExecutionContext
import org.log4s._
object OJoex {
private [this] val logger = getLogger
private[this] val logger = getLogger
def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = {
def notifyAll[F[_]: ConcurrentEffect](
store: Store[F],
clientExecutionContext: ExecutionContext
): F[Unit] =
for {
nodes <- store.transact(RNode.findAll(NodeType.Joex))
_ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext))
} yield ()
}
def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] =
def cancelJob[F[_]: ConcurrentEffect](
jobId: Ident,
worker: Ident,
store: Store[F],
clientEc: ExecutionContext
): F[Boolean] =
for {
node <- store.transact(RNode.findById(worker))
cancel <- node.traverse(joexCancel(clientEc)(_, jobId))
} yield cancel.getOrElse(false)
private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = {
val notifyUrl = node.url/"api"/"v1"/"job"/job.id/"cancel"
private def joexCancel[F[_]: ConcurrentEffect](
ec: ExecutionContext
)(node: RNode, job: Ident): F[Boolean] = {
val notifyUrl = node.url / "api" / "v1" / "job" / job.id / "cancel"
BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => true)
@ -38,7 +46,7 @@ object OJoex {
}
private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = {
val notifyUrl = node.url/"api"/"v1"/"notify"
val notifyUrl = node.url / "api" / "v1" / "notify"
val execute = BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => ())

View File

@ -36,13 +36,15 @@ object OOrganization {
case class PersonAndContacts(person: RPerson, contacts: Seq[RContact])
def apply[F[_] : Effect](store: Store[F]): Resource[F, OOrganization[F]] =
def apply[F[_]: Effect](store: Store[F]): Resource[F, OOrganization[F]] =
Resource.pure(new OOrganization[F] {
def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] =
store.transact(QOrganization.findOrgAndContact(account.collective, _.name)).
map({ case (org, cont) => OrgAndContacts(org, cont) }).
compile.toVector
store
.transact(QOrganization.findOrgAndContact(account.collective, _.name))
.map({ case (org, cont) => OrgAndContacts(org, cont) })
.compile
.toVector
def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(ROrganization.findAllRef(account.collective, _.name))
@ -54,9 +56,11 @@ object OOrganization {
QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store)
def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] =
store.transact(QOrganization.findPersonAndContact(account.collective, _.name)).
map({ case (person, cont) => PersonAndContacts(person, cont) }).
compile.toVector
store
.transact(QOrganization.findPersonAndContact(account.collective, _.name))
.map({ case (person, cont) => PersonAndContacts(person, cont) })
.compile
.toVector
def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(RPerson.findAllRef(account.collective, _.name))
@ -68,14 +72,13 @@ object OOrganization {
QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store)
def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deleteOrg(orgId, collective)).
attempt.
map(AddResult.fromUpdate)
store.transact(QOrganization.deleteOrg(orgId, collective)).attempt.map(AddResult.fromUpdate)
def deletePerson(personId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deletePerson(personId, collective)).
attempt.
map(AddResult.fromUpdate)
store
.transact(QOrganization.deletePerson(personId, collective))
.attempt
.map(AddResult.fromUpdate)
})
}

View File

@ -41,8 +41,6 @@ object OSource {
}
def delete(id: Ident, collective: Ident): F[AddResult] =
store.transact(RSource.delete(id, collective)).
attempt.
map(AddResult.fromUpdate)
store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
})
}

View File

@ -17,7 +17,6 @@ trait OTag[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult]
}
object OTag {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] =
@ -47,10 +46,7 @@ object OTag {
n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective))
} yield n0.getOrElse(0) + n1.getOrElse(0)
store.transact(io).
attempt.
map(AddResult.fromUpdate)
store.transact(io).attempt.map(AddResult.fromUpdate)
}
})
}

View File

@ -22,17 +22,27 @@ trait OUpload[F[_]] {
}
object OUpload {
private [this] val logger = getLogger
private[this] val logger = getLogger
case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte])
case class File[F[_]](
name: Option[String],
advertisedMime: Option[MimeType],
data: Stream[F, Byte]
)
case class UploadMeta( direction: Option[Direction]
, sourceAbbrev: String
, validFileTypes: Seq[MimeType])
case class UploadMeta(
direction: Option[Direction],
sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
case class UploadData[F[_]]( multiple: Boolean
, meta: UploadMeta
, files: Vector[File[F]], priority: Priority, tracker: Option[Ident])
case class UploadData[F[_]](
multiple: Boolean,
meta: UploadMeta,
files: Vector[File[F]],
priority: Priority,
tracker: Option[Ident]
)
sealed trait UploadResult
object UploadResult {
@ -41,22 +51,33 @@ object OUpload {
case object NoSource extends UploadResult
}
def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] =
def apply[F[_]: ConcurrentEffect](
store: Store[F],
queue: JobQueue[F],
cfg: Config,
httpClientEC: ExecutionContext
): Resource[F, OUpload[F]] =
Resource.pure(new OUpload[F] {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] =
for {
files <- data.files.traverse(saveFile).map(_.flatten)
pred <- checkFileList(files)
lang <- store.transact(RCollective.findLanguage(account.collective))
meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList))
meta = ProcessItemArgs.ProcessMeta(
account.collective,
lang.getOrElse(Language.German),
data.meta.direction,
data.meta.sourceAbbrev,
data.meta.validFileTypes
)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
else Vector(ProcessItemArgs(meta, files.toList))
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
_ <- logger.fdebug(s"Storing jobs: $job")
res <- job.traverse(submitJobs)
_ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective))
} yield res.fold(identity, identity)
}
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for {
@ -67,30 +88,47 @@ object OUpload {
result <- accId.traverse(acc => submit(updata, acc))
} yield result.fold(identity, identity)
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = {
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] =
for {
_ <- logger.fdebug(s"Storing jobs: $jobs")
_ <- queue.insertAll(jobs)
_ <- OJoex.notifyAll(store, httpClientEC)
} yield UploadResult.Success
}
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = {
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
logger.finfo(s"Receiving file $file") *>
store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None).
compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt.
map(_.fold(ex => {
store.bitpeace
.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None)
.compile
.lastOrError
.map(fm => Ident.unsafe(fm.id))
.attempt
.map(_.fold(ex => {
logger.warn(ex)(s"Could not store file for processing!")
None
}, id => Some(ProcessItemArgs.File(file.name, id))))
}
private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] =
Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(()))
private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = {
private def makeJobs(
args: Vector[ProcessItemArgs],
account: AccountId,
prio: Priority,
tracker: Option[Ident]
): F[Vector[RJob]] = {
def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob =
RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker)
RJob.newJob(
id,
ProcessItemArgs.taskName,
account.collective,
arg,
arg.makeSubject,
now,
account.user,
prio,
tracker
)
for {
id <- Ident.randomId[F]

View File

@ -21,19 +21,19 @@ trait OSignup[F[_]] {
object OSignup {
private[this] val logger = getLogger
def apply[F[_]:Effect](store: Store[F]): Resource[F, OSignup[F]] =
def apply[F[_]: Effect](store: Store[F]): Resource[F, OSignup[F]] =
Resource.pure(new OSignup[F] {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
if (cfg.mode == Config.Mode.Invite) {
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F]
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
NewInviteResult.passwordMismatch.pure[F]
else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id))
} else {
Effect[F].pure(NewInviteResult.invitationClosed)
}
}
def register(cfg: Config)(data: RegisterData): F[SignupResult] = {
def register(cfg: Config)(data: RegisterData): F[SignupResult] =
cfg.mode match {
case Config.Mode.Open =>
addUser(data).map(SignupResult.fromAddResult)
@ -61,7 +61,6 @@ object OSignup {
SignupResult.invalidInvitationKey.pure[F]
}
}
}
private def retryInvite(res: SignupResult): Boolean =
res match {
@ -77,22 +76,30 @@ object OSignup {
false
}
private def addUser(data: RegisterData): F[AddResult] = {
def toRecords: F[(RCollective, RUser)] =
for {
id2 <- Ident.randomId[F]
now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now)
u = RUser(
id2,
data.login,
data.collName,
PasswordCrypt.crypt(data.password),
UserState.Active,
None,
0,
None,
now
)
} yield (c, u)
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = {
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] =
for {
n1 <- RCollective.insert(coll)
n2 <- RUser.insert(user)
} yield n1 + n2
}
def collectiveExists: ConnectionIO[Boolean] =
RCollective.existsById(data.collName)

View File

@ -2,9 +2,7 @@ package docspell.backend.signup
import docspell.store.AddResult
sealed trait SignupResult {
}
sealed trait SignupResult {}
object SignupResult {

View File

@ -1,12 +1,14 @@
package docspell.common
case class Banner( component: String
, version: String
, gitHash: Option[String]
, jdbcUrl: LenientUri
, configFile: Option[String]
, appId: Ident
, baseUrl: LenientUri) {
case class Banner(
component: String,
version: String,
gitHash: Option[String],
jdbcUrl: LenientUri,
configFile: Option[String],
appId: Ident,
baseUrl: LenientUri
) {
private val banner =
"""______ _ _
@ -21,12 +23,12 @@ case class Banner( component: String
def render(prefix: String): String = {
val text = banner.split('\n').toList ++ List(
s"<< $component >>"
, s"Id: ${appId.id}"
, s"Base-Url: ${baseUrl.asString}"
, s"Database: ${jdbcUrl.asString}"
, s"Config: ${configFile.getOrElse("")}"
, ""
s"<< $component >>",
s"Id: ${appId.id}",
s"Base-Url: ${baseUrl.asString}",
s"Database: ${jdbcUrl.asString}",
s"Config: ${configFile.getOrElse("")}",
""
)
text.map(line => s"$prefix $line").mkString("\n")

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe._
import io.circe.generic.semiauto._
case class IdRef(id: Ident, name: String) {
}
case class IdRef(id: Ident, name: String) {}
object IdRef {

View File

@ -32,4 +32,3 @@ object ItemState {
implicit val jsonEncoder: Encoder[ItemState] =
Encoder.encodeString.contramap(_.name)
}

View File

@ -31,14 +31,12 @@ object Language {
def fromString(str: String): Either[String, Language] = {
val lang = str.toLowerCase
all.find(_.allNames.contains(lang)).
toRight(s"Unsupported or invalid language: $str")
all.find(_.allNames.contains(lang)).toRight(s"Unsupported or invalid language: $str")
}
def unsafe(str: String): Language =
fromString(str).fold(sys.error, identity)
implicit val jsonDecoder: Decoder[Language] =
Decoder.decodeString.emap(fromString)
implicit val jsonEncoder: Encoder[Language] =

View File

@ -12,7 +12,7 @@ case class MimeType(primary: String, sub: String) {
def matches(other: MimeType): Boolean =
primary == other.primary &&
(sub == other.sub || sub == "*" )
(sub == other.sub || sub == "*")
}
object MimeType {
@ -26,9 +26,10 @@ object MimeType {
def image(sub: String): MimeType =
MimeType("image", partFromString(sub).throwLeft)
private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
private[this] val validChars: Set[Char] =
(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
def parse(str: String): Either[String, MimeType] = {
def parse(str: String): Either[String, MimeType] =
str.indexOf('/') match {
case -1 => Left(s"Invalid MIME type: $str")
case n =>
@ -37,7 +38,6 @@ object MimeType {
sub <- partFromString(str.substring(n + 1))
} yield MimeType(prim.toLowerCase, sub.toLowerCase)
}
}
def unsafe(str: String): MimeType =
parse(str).throwLeft

View File

@ -2,6 +2,4 @@ package docspell.common
import java.time.LocalDate
case class NerDateLabel(date: LocalDate, label: NerLabel) {
}
case class NerDateLabel(date: LocalDate, label: NerLabel) {}

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {
}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {}
object NerLabel {
implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel]

View File

@ -4,7 +4,7 @@ import io.circe.{Decoder, Encoder}
final class Password(val pass: String) extends AnyVal {
def isEmpty: Boolean= pass.isEmpty
def isEmpty: Boolean = pass.isEmpty
override def toString: String =
if (pass.isEmpty) "<empty>" else "***"

View File

@ -16,7 +16,6 @@ object Priority {
case object Low extends Priority
def fromString(str: String): Either[String, Priority] =
str.toLowerCase match {
case "high" => Right(High)
@ -27,7 +26,6 @@ object Priority {
def unsafe(str: String): Priority =
fromString(str).fold(sys.error, identity)
def fromInt(n: Int): Priority =
if (n <= toInt(Low)) Low
else High

View File

@ -6,14 +6,13 @@ import ProcessItemArgs._
case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
def makeSubject: String = {
def makeSubject: String =
files.flatMap(_.name) match {
case Nil => s"${meta.sourceAbbrev}: No files"
case n :: Nil => n
case n1 :: n2 :: Nil => s"$n1, $n2"
case _ => s"${files.size} files from ${meta.sourceAbbrev}"
}
}
}
@ -21,11 +20,13 @@ object ProcessItemArgs {
val taskName = Ident.unsafe("process-item")
case class ProcessMeta( collective: Ident
, language: Language
, direction: Option[Direction]
, sourceAbbrev: String
, validFileTypes: Seq[MimeType])
case class ProcessMeta(
collective: Ident,
language: Language,
direction: Option[Direction],
sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
object ProcessMeta {
implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta]

View File

@ -30,8 +30,6 @@ object Timestamp {
def current[F[_]: Sync]: F[Timestamp] =
Sync[F].delay(Timestamp(Instant.now))
implicit val encodeTimestamp: Encoder[Timestamp] =
BaseJsonCodecs.encodeInstantEpoch.contramap(_.value)

View File

@ -12,7 +12,6 @@ object UserState {
/** The user is blocked by an admin. */
case object Disabled extends UserState
def fromString(s: String): Either[String, UserState] =
s.toLowerCase match {
case "active" => Right(Active)

View File

@ -11,13 +11,18 @@ trait StreamSyntax {
implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
s.fold("")(_ + _).
compile.last.
map(optStr => for {
str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value"))
s.fold("")(_ + _)
.compile
.last
.map(optStr =>
for {
str <- optStr
.map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying)
value <- json.as[A]
} yield value)
} yield value
)
}

View File

@ -5,17 +5,23 @@ import docspell.joex.scheduler.SchedulerConfig
import docspell.store.JdbcConfig
import docspell.text.ocr.{Config => OcrConfig}
case class Config(appId: Ident
, baseUrl: LenientUri
, bind: Config.Bind
, jdbc: JdbcConfig
, scheduler: SchedulerConfig
, extraction: OcrConfig
case class Config(
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
jdbc: JdbcConfig,
scheduler: SchedulerConfig,
extraction: OcrConfig
)
object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "")
val postgres =
JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
case class Bind(address: String, port: Int)
}

View File

@ -11,7 +11,6 @@ object ConfigFile {
def loadConfig: Config =
ConfigSource.default.at("docspell.joex").loadOrThrow[Config]
object Implicits {
implicit val countingSchemeReader: ConfigReader[CountingScheme] =
ConfigReader[String].emap(reason(CountingScheme.readString))

View File

@ -12,11 +12,13 @@ import fs2.concurrent.SignallingRef
import scala.concurrent.ExecutionContext
final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config
, nodeOps: ONode[F]
, store: Store[F]
, termSignal: SignallingRef[F, Boolean]
, val scheduler: Scheduler[F]) extends JoexApp[F] {
final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer](
cfg: Config,
nodeOps: ONode[F],
store: Store[F],
termSignal: SignallingRef[F, Boolean],
val scheduler: Scheduler[F]
) extends JoexApp[F] {
def init: F[Unit] = {
val run = scheduler.start.compile.drain
@ -40,16 +42,24 @@ final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Conf
object JoexAppImpl {
def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config
, termSignal: SignallingRef[F, Boolean]
, connectEC: ExecutionContext
, blocker: Blocker): Resource[F, JoexApp[F]] =
def create[F[_]: ConcurrentEffect: ContextShift: Timer](
cfg: Config,
termSignal: SignallingRef[F, Boolean],
connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, JoexApp[F]] =
for {
store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store).
withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])).
resource
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
.withTask(
JobTask.json(
ProcessItemArgs.taskName,
ItemHandler[F](cfg.extraction),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR

View File

@ -15,11 +15,17 @@ import scala.concurrent.ExecutionContext
object JoexServer {
private case class App[F[_]](
httpApp: HttpApp[F],
termSig: SignallingRef[F, Boolean],
exitRef: Ref[F, ExitCode]
)
private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode])
def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker)
(implicit T: Timer[F]): Stream[F, Nothing] = {
def stream[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F]): Stream[F, Nothing] = {
val app = for {
signal <- Resource.liftF(SignallingRef[F, Boolean](false))
@ -36,13 +42,14 @@ object JoexServer {
} yield App(finalHttpApp, signal, exitCode)
Stream.resource(app).flatMap(app =>
BlazeServerBuilder[F].
bindHttp(cfg.bind.port, cfg.bind.address).
withHttpApp(app.httpApp).
withoutBanner.
serveWhile(app.termSig, app.exitRef)
Stream
.resource(app)
.flatMap(app =>
BlazeServerBuilder[F]
.bindHttp(cfg.bind.port, cfg.bind.address)
.withHttpApp(app.httpApp)
.withoutBanner
.serveWhile(app.termSig, app.exitRef)
)
}.drain

View File

@ -14,10 +14,12 @@ object Main extends IOApp {
private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking")))
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect")))
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))
)
def run(args: List[String]) = {
args match {
@ -40,12 +42,15 @@ object Main extends IOApp {
}
val cfg = ConfigFile.loadConfig
val banner = Banner("JOEX"
, BuildInfo.version
, BuildInfo.gitHeadCommit
, cfg.jdbc.url
, Option(System.getProperty("config.file"))
, cfg.appId, cfg.baseUrl)
val banner = Banner(
"JOEX",
BuildInfo.version,
BuildInfo.gitHeadCommit,
cfg.jdbc.url,
Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}")
JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success)
}

View File

@ -23,20 +23,28 @@ object CreateItem {
Task { ctx =>
val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet
def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files).
flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))).
collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }).
zipWithIndex.
evalMap({ case (f, index) =>
Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
}).
compile.toVector
def fileMetas(itemId: Ident, now: Timestamp) =
Stream
.emits(ctx.args.files)
.flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
.collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f })
.zipWithIndex
.evalMap({
case (f, index) =>
Ident
.randomId[F]
.map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
})
.compile
.toVector
val item = RItem.newItem[F](ctx.args.meta.collective
, ctx.args.makeSubject
, ctx.args.meta.sourceAbbrev
, ctx.args.meta.direction.getOrElse(Direction.Incoming)
, ItemState.Premature)
val item = RItem.newItem[F](
ctx.args.meta.collective,
ctx.args.makeSubject,
ctx.args.meta.sourceAbbrev,
ctx.args.meta.direction.getOrElse(Direction.Incoming),
ItemState.Premature
)
for {
_ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)")
@ -56,16 +64,28 @@ object CreateItem {
Task { ctx =>
for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F]
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F]
rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid)))
} yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
else ().pure[F]
rms <- cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
} yield cand.headOption.map(ri =>
ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)
)
}
private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] =
private def logDifferences[F[_]: Sync](
ctx: Context[F, ProcessItemArgs],
saved: Vector[RAttachment],
saveCount: Int
): F[Unit] =
if (ctx.args.files.size != saved.size) {
ctx.logger.warn(s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount")
ctx.logger.warn(
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
)
} else {
().pure[F]
}

View File

@ -19,45 +19,65 @@ object FindProposal {
def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
val rmas = data.metas.map(rm =>
rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
ctx.logger.info("Starting find-proposal") *>
rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))).
flatMap(rmv => rmv.traverse(rm =>
rmas
.traverse(rm =>
processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))
)
.flatMap(rmv =>
rmv
.traverse(rm =>
ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *>
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))).
map(_ => data.copy(metas = rmv)))
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))
)
.map(_ => data.copy(metas = rmv))
)
}
def processAttachment[F[_]: Sync]( rm: RAttachmentMeta
, rd: Vector[NerDateLabel]
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
def processAttachment[F[_]: Sync](
rm: RAttachmentMeta,
rd: Vector[NerDateLabel],
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx))
List(finder.find(rm.nerlabels), makeDateProposal(rd)).
traverse(identity).map(MetaProposalList.flatten)
List(finder.find(rm.nerlabels), makeDateProposal(rd))
.traverse(identity)
.map(MetaProposalList.flatten)
}
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = {
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] =
Timestamp.current[F].map { now =>
val latestFirst = dates.sortWith(_.date isAfter _.date)
val latestFirst = dates.sortWith((l1, l2) => l1.date.isAfter(l2.date))
val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate
val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate))
val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate,
after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))))
val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate,
before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))))
val dueDates = MetaProposalList.fromSeq1(
MetaProposalType.DueDate,
after.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
val itemDates = MetaProposalList.fromSeq1(
MetaProposalType.DocDate,
before.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
MetaProposalList.flatten(Seq(dueDates, itemDates))
}
}
def removeDuplicates(labels: List[NerLabel]): List[NerLabel] =
labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) =>
if (seen.contains(el.tag.name+el.label.toLowerCase)) (seen, result)
labels
.foldLeft((Set.empty[String], List.empty[NerLabel])) {
case ((seen, result), el) =>
if (seen.contains(el.tag.name + el.label.toLowerCase)) (seen, result)
else (seen + (el.tag.name + el.label.toLowerCase), el :: result)
}._2.sortBy(_.startPosition)
}
._2
.sortBy(_.startPosition)
trait Finder[F[_]] { self =>
def find(labels: Seq[NerLabel]): F[MetaProposalList]
@ -80,12 +100,14 @@ object FindProposal {
else f.map(ml1 => ml0.fillEmptyFrom(ml1))
})
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)
(implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] =
flatMap(res0 => {
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
implicit F: FlatMap[F],
F2: Applicative[F]
): Finder[F] =
flatMap { res0 =>
if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0)
else f.map(res1 => res0.fillEmptyFrom(res1))
})
}
}
object Finder {
@ -102,7 +124,11 @@ object FindProposal {
labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten)
}
private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
private def search[F[_]: Sync](
nt: NerLabel,
exact: Boolean,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val value =
if (exact) normalizeSearchValue(nt.label)
else s"%${normalizeSearchValue(nt.label)}%"
@ -110,42 +136,51 @@ object FindProposal {
if (exact) 2 else 5
if (value.length < minLength) {
ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty)
} else nt.tag match {
ctx.logger
.debug(s"Skipping too small value '$value' (original '${nt.label}').")
.map(_ => MetaProposalList.empty)
} else
nt.tag match {
case NerTag.Organization =>
ctx.logger.debug(s"Looking for organizations: $value") *>
ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)).
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
ctx.store
.transact(ROrganization.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
case NerTag.Person =>
val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)).
map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val s1 = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, value, true))
.map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
val s2 = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking for persons: $value") *> (for {
ml0 <- s1
ml1 <- s2
} yield ml0 |+| ml1)
case NerTag.Location =>
ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.").
map(_ => MetaProposalList.empty)
ctx.logger
.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.")
.map(_ => MetaProposalList.empty)
case NerTag.Misc =>
ctx.logger.debug(s"Looking for equipments: $value") *>
ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)).
map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
ctx.store
.transact(REquipment.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
case NerTag.Email =>
searchContact(nt, ContactKind.Email, value, ctx)
case NerTag.Website =>
if (!exact) {
val searchString = Domain.domainFromUri(nt.label.toLowerCase).
toOption.
map(_.toPrimaryDomain.asString).
map(s => s"%$s%").
getOrElse(value)
val searchString = Domain
.domainFromUri(nt.label.toLowerCase)
.toOption
.map(_.toPrimaryDomain.asString)
.map(s => s"%$s%")
.getOrElse(value)
searchContact(nt, ContactKind.Website, searchString, ctx)
} else {
searchContact(nt, ContactKind.Website, value, ctx)
@ -157,23 +192,28 @@ object FindProposal {
}
}
private def searchContact[F[_]: Sync]( nt: NerLabel
, kind: ContactKind
, value: String
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)).
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
private def searchContact[F[_]: Sync](
nt: NerLabel,
kind: ContactKind,
value: String,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val orgs = ctx.store
.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
val corrP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking with $kind: $value") *>
List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten)
}
// The backslash *must* be stripped from search strings.
private [this] val invalidSearch =
private[this] val invalidSearch =
"…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet
private def normalizeSearchValue(str: String): String =

View File

@ -4,10 +4,12 @@ import docspell.common.{Ident, NerDateLabel, NerLabel}
import docspell.joex.process.ItemData.AttachmentDates
import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
case class ItemData( item: RItem
, attachments: Vector[RAttachment]
, metas: Vector[RAttachmentMeta]
, dateLabels: Vector[AttachmentDates]) {
case class ItemData(
item: RItem,
attachments: Vector[RAttachment],
metas: Vector[RAttachmentMeta],
dateLabels: Vector[AttachmentDates]
) {
def findMeta(attachId: Ident): Option[RAttachmentMeta] =
metas.find(_.id == attachId)
@ -16,7 +18,6 @@ case class ItemData( item: RItem
dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty)
}
object ItemData {
case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) {

View File

@ -10,14 +10,13 @@ import docspell.text.ocr.{Config => OcrConfig}
object ItemHandler {
def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
logWarn("Now cancelling. Deleting potentially created data.").
flatMap(_ => deleteByFileIds)
logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ => deleteByFileIds)
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] =
CreateItem[F].
flatMap(itemStateTask(ItemState.Processing)).
flatMap(safeProcess[F](cfg)).
map(_ => ())
CreateItem[F]
.flatMap(itemStateTask(ItemState.Processing))
.flatMap(safeProcess[F](cfg))
.map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx =>
@ -30,22 +29,21 @@ object ItemHandler {
last = ctx.config.retries == current.getOrElse(0)
} yield last
def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
def safeProcess[F[_]: Sync: ContextShift](
cfg: OcrConfig
)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
case true =>
ProcessItem[F](cfg)(data).
attempt.flatMap({
ProcessItem[F](cfg)(data).attempt.flatMap({
case Right(d) =>
Task.pure(d)
case Left(ex) =>
logWarn[F]("Processing failed on last retry. Creating item but without proposals.").
flatMap(_ => itemStateTask(ItemState.Created)(data)).
andThen(_ => Sync[F].raiseError(ex))
logWarn[F]("Processing failed on last retry. Creating item but without proposals.")
.flatMap(_ => itemStateTask(ItemState.Created)(data))
.andThen(_ => Sync[F].raiseError(ex))
})
case false =>
ProcessItem[F](cfg)(data).
flatMap(itemStateTask(ItemState.Created))
ProcessItem[F](cfg)(data).flatMap(itemStateTask(ItemState.Created))
}
def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =

View File

@ -13,28 +13,40 @@ object LinkProposal {
val proposals = MetaProposalList.flatten(data.metas.map(_.proposals))
ctx.logger.info(s"Starting linking proposals") *>
MetaProposalType.all.
traverse(applyValue(data, proposals, ctx)).
map(result => ctx.logger.info(s"Results from proposal processing: $result")).
map(_ => data)
MetaProposalType.all
.traverse(applyValue(data, proposals, ctx))
.map(result => ctx.logger.info(s"Results from proposal processing: $result"))
.map(_ => data)
}
def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = {
def applyValue[F[_]: Sync](
data: ItemData,
proposalList: MetaProposalList,
ctx: Context[F, ProcessItemArgs]
)(mpt: MetaProposalType): F[Result] =
proposalList.find(mpt) match {
case None =>
Result.noneFound(mpt).pure[F]
case Some(a) if a.isSingleValue =>
ctx.logger.info(s"Found one candidate for ${a.proposalType}") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).
map(_ => Result.single(mpt))
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.single(mpt)
)
case Some(a) =>
ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).
map(_ => Result.multiple(mpt))
}
ctx.logger.info(
s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first."
) *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.multiple(mpt)
)
}
def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] =
def setItemMeta[F[_]: Sync](
itemId: Ident,
ctx: Context[F, ProcessItemArgs],
mpt: MetaProposalType,
value: Ident
): F[Int] =
mpt match {
case MetaProposalType.CorrOrg =>
ctx.logger.debug(s"Updating item organization with: ${value.id}") *>
@ -54,7 +66,6 @@ object LinkProposal {
ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0)
}
sealed trait Result {
def proposalType: MetaProposalType
}

View File

@ -7,13 +7,15 @@ import docspell.text.ocr.{Config => OcrConfig}
object ProcessItem {
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
TextExtraction(cfg, item).
flatMap(Task.setProgress(25)).
flatMap(TextAnalysis[F]).
flatMap(Task.setProgress(50)).
flatMap(FindProposal[F]).
flatMap(Task.setProgress(75)).
flatMap(LinkProposal[F]).
flatMap(Task.setProgress(99))
def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
TextExtraction(cfg, item)
.flatMap(Task.setProgress(25))
.flatMap(TextAnalysis[F])
.flatMap(Task.setProgress(50))
.flatMap(FindProposal[F])
.flatMap(Task.setProgress(75))
.flatMap(LinkProposal[F])
.flatMap(Task.setProgress(99))
}

View File

@ -8,7 +8,7 @@ import docspell.joex.scheduler.Task
import org.log4s._
object TestTasks {
private [this] val logger = getLogger
private[this] val logger = getLogger
def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
Task { ctx =>
@ -17,9 +17,9 @@ object TestTasks {
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx =>
ctx.logger.info(s"Failing the task run :(").map(_ =>
sys.error("Oh, cannot extract gold from this document")
)
ctx.logger
.info(s"Failing the task run :(")
.map(_ => sys.error("Oh, cannot extract gold from this document"))
}
def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
@ -34,6 +34,6 @@ object TestTasks {
ctx.logger.info("doing more things")
}
private def sleep[F[_]:Sync](ms: Long): F[Unit] =
private def sleep[F[_]: Sync](ms: Long): F[Unit] =
Sync[F].delay(Thread.sleep(ms))
}

View File

@ -19,21 +19,26 @@ object TextAnalysis {
s <- Duration.stopTime[F]
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)))
_ <- t.traverse(m =>
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
)
e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector
} yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2))
}
def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
def annotateAttachment[F[_]: Sync](
lang: Language
)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
for {
list0 <- stanfordNer[F](lang, rm)
list1 <- contactNer[F](rm)
dates <- dateNer[F](rm, lang)
} yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates)
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay {
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] =
Sync[F].delay {
rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty)
}
@ -42,8 +47,10 @@ object TextAnalysis {
}
def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay {
AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty))
AttachmentDates(
rm,
rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)
)
}
}

View File

@ -11,7 +11,10 @@ import docspell.text.ocr.{TextExtract, Config => OcrConfig}
object TextExtraction {
def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] =
def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig,
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
for {
_ <- ctx.logger.info("Starting text extraction")
@ -24,22 +27,33 @@ object TextExtraction {
} yield item.copy(metas = txt)
}
def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] =
def extractTextToMeta[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: OcrConfig,
lang: Language
)(ra: RAttachment): F[RAttachmentMeta] =
for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}")
dst <- Duration.stopTime[F]
txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra)
meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty))
est <- dst
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}")
_ <- ctx.logger.debug(
s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}"
)
} yield meta
def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace.get(ra.fileId.id).
unNoneTerminate.
through(store.bitpeace.fetchData2(RangeDef.all))
def extractText[F[_]: Sync: ContextShift](
ocrConfig: OcrConfig,
lang: Language,
store: Store[F],
blocker: Blocker
)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.through(store.bitpeace.fetchData2(RangeDef.all))
TextExtract.extract(data, blocker, lang.iso3, ocrConfig).
compile.last
TextExtract.extract(data, blocker, lang.iso3, ocrConfig).compile.last
}
}

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of[F] {
case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version
, BuildInfo.builtAtMillis
, BuildInfo.builtAtString
, BuildInfo.gitHeadCommit.getOrElse("")
, BuildInfo.gitDescribedVersion.getOrElse("")))
Ok(
VersionInfo(
BuildInfo.version,
BuildInfo.builtAtMillis,
BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
}
}
}

View File

@ -13,7 +13,7 @@ import org.http4s.dsl.Http4sDsl
object JoexRoutes {
def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of[F] {
case POST -> Root / "notify" =>
@ -31,7 +31,9 @@ object JoexRoutes {
case POST -> Root / "shutdownAndExit" =>
for {
_ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown)
_ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp
@ -52,7 +54,15 @@ object JoexRoutes {
}
def mkJob(j: RJob): Job =
Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch))
Job(
j.id,
j.subject,
j.submitted,
j.priority,
j.retries,
j.progress,
j.started.getOrElse(Timestamp.Epoch)
)
def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog =
JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList)

View File

@ -30,22 +30,26 @@ trait Context[F[_], A] { self =>
}
object Context {
private [this] val log = getLogger
private[this] val log = getLogger
def create[F[_]: Functor, A]( job: RJob
, arg: A
, config: SchedulerConfig
, log: Logger[F]
, store: Store[F]
, blocker: Blocker): Context[F, A] =
def create[F[_]: Functor, A](
job: RJob,
arg: A,
config: SchedulerConfig,
log: Logger[F],
store: Store[F],
blocker: Blocker
): Context[F, A] =
new ContextImpl(arg, log, store, blocker, config, job.id)
def apply[F[_]: Concurrent, A]( job: RJob
, arg: A
, config: SchedulerConfig
, logSink: LogSink[F]
, blocker: Blocker
, store: Store[F]): F[Context[F, A]] =
def apply[F[_]: Concurrent, A](
job: RJob,
arg: A,
config: SchedulerConfig,
logSink: LogSink[F],
blocker: Blocker,
store: Store[F]
): F[Context[F, A]] =
for {
_ <- log.ftrace("Creating logger for task run")
logger <- Logger(job.id, job.info, config.logBufferSize, logSink)
@ -53,13 +57,14 @@ object Context {
ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx
private final class ContextImpl[F[_]: Functor, A]( val args: A
, val logger: Logger[F]
, val store: Store[F]
, val blocker: Blocker
, val config: SchedulerConfig
, val jobId: Ident)
extends Context[F,A] {
final private class ContextImpl[F[_]: Functor, A](
val args: A,
val logger: Logger[F],
val store: Store[F],
val blocker: Blocker,
val config: SchedulerConfig,
val jobId: Ident
) extends Context[F, A] {
def setProgress(percent: Int): F[Unit] = {
val pval = math.min(100, math.max(0, percent))

View File

@ -11,14 +11,13 @@ import docspell.common.Priority
*/
case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
def nextPriority: (CountingScheme, Priority) = {
def nextPriority: (CountingScheme, Priority) =
if (counter <= 0) (increment, Priority.High)
else {
val rest = counter % (high + low)
if (rest < high) (increment, Priority.High)
else (increment, Priority.Low)
}
}
def increment: CountingScheme =
copy(counter = counter + 1)
@ -32,8 +31,7 @@ object CountingScheme {
def readString(str: String): Either[String, CountingScheme] =
str.split(',') match {
case Array(h, l) =>
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).
left.map(_.getMessage)
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage)
case _ =>
Left(s"Invalid counting scheme: $str")
}

View File

@ -20,12 +20,15 @@ case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Tas
object JobTask {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])
(implicit D: Decoder[A]): JobTask[F] = {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])(
implicit D: Decoder[A]
): JobTask[F] = {
val convert: String => F[A] =
str => str.parseJsonAs[A] match {
str =>
str.parseJsonAs[A] match {
case Right(a) => a.pure[F]
case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
case Left(ex) =>
Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
}
JobTask(name, task.contramap(convert), onCancel.contramap(convert))

View File

@ -4,12 +4,14 @@ import cats.implicits._
import docspell.common._
import cats.effect.Sync
case class LogEvent( jobId: Ident
, jobInfo: String
, time: Timestamp
, level: LogLevel
, msg: String
, ex: Option[Throwable] = None) {
case class LogEvent(
jobId: Ident,
jobInfo: String,
time: Timestamp,
level: LogLevel,
msg: String,
ex: Option[Throwable] = None
) {
def logLine: String =
s">>> ${time.asString} $level $jobInfo: $msg"
@ -21,5 +23,4 @@ object LogEvent {
def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] =
Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg))
}

View File

@ -44,12 +44,22 @@ object LogSink {
LogSink(_.evalMap(e => logInternal(e)))
def db[F[_]: Sync](store: Store[F]): LogSink[F] =
LogSink(_.evalMap(ev => for {
LogSink(
_.evalMap(ev =>
for {
id <- Ident.randomId[F]
joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse(""))
joblog = RJobLog(
id,
ev.jobId,
ev.level,
ev.time,
ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("")
)
_ <- logInternal(ev)
_ <- store.transact(RJobLog.insert(joblog))
} yield ()))
} yield ()
)
)
def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = {
val s: Stream[F, Pipe[F, LogEvent, Unit]] =

View File

@ -33,13 +33,21 @@ object Logger {
LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1)
def error(ex: Throwable)(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1)
LogEvent
.create[F](jobId, jobInfo, LogLevel.Error, msg)
.map(le => le.copy(ex = Some(ex)))
.flatMap(q.enqueue1)
def error(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1)
}
def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] =
def apply[F[_]: Concurrent](
jobId: Ident,
jobInfo: String,
bufferSize: Int,
sink: LogSink[F]
): F[Logger[F]] =
for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q)

View File

@ -7,13 +7,14 @@ import docspell.store.Store
import docspell.store.queue.JobQueue
import fs2.concurrent.SignallingRef
case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
config: SchedulerConfig
, tasks: JobTaskRegistry[F]
, store: Store[F]
, blocker: Blocker
, queue: Resource[F, JobQueue[F]]
, logSink: LogSink[F]) {
case class SchedulerBuilder[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig,
tasks: JobTaskRegistry[F],
store: Store[F],
blocker: Blocker,
queue: Resource[F, JobQueue[F]],
logSink: LogSink[F]
) {
def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] =
copy(config = cfg)
@ -33,7 +34,6 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] =
copy(logSink = sink)
def serve: Resource[F, Scheduler[F]] =
resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch))
@ -45,22 +45,25 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
perms <- Resource.liftF(Semaphore(config.poolSize.toLong))
} yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms)
scheduler.evalTap(_.init).
map(s => s: Scheduler[F])
scheduler.evalTap(_.init).map(s => s: Scheduler[F])
}
}
object SchedulerBuilder {
def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig
, blocker: Blocker
, store: Store[F]): SchedulerBuilder[F] =
new SchedulerBuilder[F](config
, JobTaskRegistry.empty[F]
, store
, blocker
, JobQueue(store)
, LogSink.db[F](store))
def apply[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig,
blocker: Blocker,
store: Store[F]
): SchedulerBuilder[F] =
new SchedulerBuilder[F](
config,
JobTaskRegistry.empty[F],
store,
blocker,
JobQueue(store),
LogSink.db[F](store)
)
}

View File

@ -2,24 +2,26 @@ package docspell.joex.scheduler
import docspell.common._
case class SchedulerConfig( name: Ident
, poolSize: Int
, countingScheme: CountingScheme
, retries: Int
, retryDelay: Duration
, logBufferSize: Int
, wakeupPeriod: Duration
)
case class SchedulerConfig(
name: Ident,
poolSize: Int,
countingScheme: CountingScheme,
retries: Int,
retryDelay: Duration,
logBufferSize: Int,
wakeupPeriod: Duration
)
object SchedulerConfig {
val default = SchedulerConfig(
name = Ident.unsafe("default-scheduler")
, poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
, countingScheme = CountingScheme(2, 1)
, retries = 5
, retryDelay = Duration.seconds(30)
, logBufferSize = 500
, wakeupPeriod = Duration.minutes(10)
name = Ident.unsafe("default-scheduler"),
poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
,
countingScheme = CountingScheme(2, 1),
retries = 5,
retryDelay = Duration.seconds(30),
logBufferSize = 500,
wakeupPeriod = Duration.minutes(10)
)
}

View File

@ -14,17 +14,19 @@ import SchedulerImpl._
import docspell.store.Store
import docspell.store.queries.QJob
final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig
, blocker: Blocker
, queue: JobQueue[F]
, tasks: JobTaskRegistry[F]
, store: Store[F]
, logSink: LogSink[F]
, state: SignallingRef[F, State[F]]
, waiter: SignallingRef[F, Boolean]
, permits: Semaphore[F]) extends Scheduler[F] {
final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
val config: SchedulerConfig,
blocker: Blocker,
queue: JobQueue[F],
tasks: JobTaskRegistry[F],
store: Store[F],
logSink: LogSink[F],
state: SignallingRef[F, State[F]],
waiter: SignallingRef[F, Boolean],
permits: Semaphore[F]
) extends Scheduler[F] {
private [this] val logger = getLogger
private[this] val logger = getLogger
/**
* On startup, get all jobs in state running from this scheduler
@ -34,8 +36,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.runningToWaiting(config.name, store)
def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] =
ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala).
evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain)
ConcurrentEffect[F].start(
Stream
.awakeEvery[F](config.wakeupPeriod.toScala)
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
.compile
.drain
)
def getRunning: F[Vector[RJob]] =
state.get.flatMap(s => QJob.findAll(s.getRunning, store))
@ -51,21 +58,23 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def shutdown(cancelAll: Boolean): F[Unit] = {
val doCancel =
state.get.
flatMap(_.cancelTokens.values.toList.traverse(identity)).
map(_ => ())
state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ())
val runShutdown =
state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F])
val wait = Stream.eval(runShutdown).
evalMap(_ => logger.finfo("Scheduler is shutting down now.")).
flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))).
flatMap(state => {
val wait = Stream
.eval(runShutdown)
.evalMap(_ => logger.finfo("Scheduler is shutting down now."))
.flatMap(_ =>
Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
)
.flatMap { state =>
if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running."))
else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
else
Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
Stream.emit(state)
})
}
(wait.drain ++ Stream.emit(())).compile.lastOrError
}
@ -82,15 +91,24 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
_ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay)
else
queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined
Stream.eval(state.get.map(_.shutdownRequest)).
evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]).
flatMap(if (_) Stream.empty else Stream.eval(body)).
flatMap({
Stream
.eval(state.get.map(_.shutdownRequest))
.evalTap(
if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
else ().pure[F]
)
.flatMap(if (_) Stream.empty else Stream.eval(body))
.flatMap({
case true =>
mainLoop
case false =>
@ -103,7 +121,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def execute(job: RJob): F[Unit] = {
val task = for {
jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}")
jobtask <- tasks
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask
task match {
@ -123,7 +143,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onFinish(job: RJob, finalState: JobState): F[Unit] =
for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)"))
_ <- permits.release *> permits.available.flatMap(a =>
logger.fdebug(s"Permit released ($a free)")
)
_ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store)
} yield ()
@ -131,9 +153,14 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onStart(job: RJob): F[Unit] =
QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck
def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = {
task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)).
mapF(_.attempt.flatMap({
def wrapTask(
job: RJob,
task: Task[F, String, Unit],
ctx: Context[F, String]
): Task[F, String, Unit] =
task
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa))
.mapF(_.attempt.flatMap({
case Right(()) =>
logger.info(s"Job execution successful: ${job.info}")
ctx.logger.info("Job execution successful") *>
@ -148,16 +175,18 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.exceedsRetries(job.id, config.retries, store).flatMap {
case true =>
logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.").
map(_ => JobState.Failed: JobState)
ctx.logger
.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
.map(_ => JobState.Failed: JobState)
case false =>
logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.").
map(_ => JobState.Stuck: JobState)
ctx.logger
.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
.map(_ => JobState.Stuck: JobState)
}
}
})).
mapF(_.attempt.flatMap {
}))
.mapF(_.attempt.flatMap {
case Right(jstate) =>
onFinish(job, jstate)
case Left(ex) =>
@ -166,13 +195,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
// since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways
onFinish(job, JobState.Stuck)
})
}
def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = {
val bfa = blocker.blockOn(code)
logger.fdebug(s"Forking job ${job.info}") *>
ConcurrentEffect[F].start(bfa).
map(fiber =>
ConcurrentEffect[F]
.start(bfa)
.map(fiber =>
logger.fdebug(s"Cancelling job ${job.info}") *>
fiber.cancel *>
onCancel.attempt.map({
@ -184,7 +213,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
state.modify(_.markCancelled(job)) *>
onFinish(job, JobState.Cancelled) *>
ctx.logger.warn("Job has been cancelled.") *>
logger.fdebug(s"Job ${job.info} has been cancelled."))
logger.fdebug(s"Job ${job.info} has been cancelled.")
)
}
}
@ -193,10 +223,12 @@ object SchedulerImpl {
def emptyState[F[_]]: State[F] =
State(Map.empty, Set.empty, Map.empty, false)
case class State[F[_]]( counters: Map[Ident, CountingScheme]
, cancelled: Set[Ident]
, cancelTokens: Map[Ident, CancelToken[F]]
, shutdownRequest: Boolean) {
case class State[F[_]](
counters: Map[Ident, CountingScheme],
cancelled: Set[Ident],
cancelTokens: Map[Ident, CancelToken[F]],
shutdownRequest: Boolean
) {
def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = {
val (cs, prio) = counters.getOrElse(group, initial).nextPriority

View File

@ -24,11 +24,11 @@ trait Task[F[_], A, B] {
def mapF[C](f: F[B] => F[C]): Task[F, A, C] =
Task(Task.toKleisli(this).mapF(f))
def attempt(implicit F: ApplicativeError[F,Throwable]): Task[F, A, Either[Throwable, B]] =
def attempt(implicit F: ApplicativeError[F, Throwable]): Task[F, A, Either[Throwable, B]] =
mapF(_.attempt)
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = {
ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { ctxc: Context[F, C] =>
f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
}
}
@ -46,7 +46,6 @@ object Task {
def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] =
c => k.run(c)
def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] =
Kleisli(t.run)

View File

@ -6,7 +6,7 @@ import minitest.SimpleTestSuite
object CountingSchemeSpec extends SimpleTestSuite {
test("counting") {
val cs = CountingScheme(2,1)
val cs = CountingScheme(2, 1)
val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2)
val expect = List(Priority.High, Priority.High, Priority.Low)
assertEquals(list, expect ++ expect)

View File

@ -7,27 +7,37 @@ import docspell.backend.{Config => BackendConfig}
import docspell.common._
import scodec.bits.ByteVector
case class Config(appName: String
, appId: Ident
, baseUrl: LenientUri
, bind: Config.Bind
, backend: BackendConfig
, auth: Login.Config
case class Config(
appName: String,
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
backend: BackendConfig,
auth: Login.Config
)
object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "")
val postgres =
JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
val default: Config =
Config("Docspell"
, Ident.unsafe("restserver1")
, LenientUri.unsafe("http://localhost:7880")
, Config.Bind("localhost", 7880)
, BackendConfig(postgres
, SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24))
, BackendConfig.Files(512 * 1024, List(MimeType.pdf)))
, Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2)))
Config(
"Docspell",
Ident.unsafe("restserver1"),
LenientUri.unsafe("http://localhost:7880"),
Config.Bind("localhost", 7880),
BackendConfig(
postgres,
SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)),
BackendConfig.Files(512 * 1024, List(MimeType.pdf))
),
Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))
)
case class Bind(address: String, port: Int)
}

View File

@ -13,12 +13,14 @@ import org.log4s._
object Main extends IOApp {
private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool(
ThreadFactories.ofName("docspell-restserver-blocking")))
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-restserver-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5,
ThreadFactories.ofName("docspell-dbconnect")))
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-dbconnect"))
)
def run(args: List[String]) = {
args match {
@ -41,12 +43,15 @@ object Main extends IOApp {
}
val cfg = ConfigFile.loadConfig
val banner = Banner("REST Server"
, BuildInfo.version
, BuildInfo.gitHeadCommit
, cfg.backend.jdbc.url
, Option(System.getProperty("config.file"))
, cfg.appId, cfg.baseUrl)
val banner = Banner(
"REST Server",
BuildInfo.version,
BuildInfo.gitHeadCommit,
cfg.backend.jdbc.url,
Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}")
RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success)
}

View File

@ -7,7 +7,8 @@ import docspell.common.NodeType
import scala.concurrent.ExecutionContext
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] {
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F])
extends RestApp[F] {
def init: F[Unit] =
backend.node.register(config.appId, NodeType.Restserver, config.baseUrl)
@ -18,7 +19,12 @@ final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[
object RestAppImpl {
def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] =
def create[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, RestApp[F]] =
for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend)

View File

@ -15,8 +15,12 @@ import scala.concurrent.ExecutionContext
object RestServer {
def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker)
(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
def stream[F[_]: ConcurrentEffect](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
val app = for {
restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker)
@ -24,8 +28,8 @@ object RestServer {
httpApp = Router(
"/api/info" -> routes.InfoRoutes(),
"/api/v1/open/" -> openRoutes(cfg, restApp),
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) {
token => securedRoutes(cfg, restApp, token)
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
securedRoutes(cfg, restApp, token)
},
"/app/assets" -> WebjarRoutes.appRoutes[F](blocker),
"/app" -> TemplateRoutes[F](blocker, cfg)
@ -35,16 +39,22 @@ object RestServer {
} yield finalHttpApp
Stream.resource(app).flatMap(httpApp =>
BlazeServerBuilder[F].
bindHttp(cfg.bind.port, cfg.bind.address).
withHttpApp(httpApp).
withoutBanner.
serve)
Stream
.resource(app)
.flatMap(httpApp =>
BlazeServerBuilder[F]
.bindHttp(cfg.bind.port, cfg.bind.address)
.withHttpApp(httpApp)
.withoutBanner
.serve
)
}.drain
def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] =
def securedRoutes[F[_]: Effect](
cfg: Config,
restApp: RestApp[F],
token: AuthToken
): HttpRoutes[F] =
Router(
"auth" -> LoginRoutes.session(restApp.backend.login, cfg),
"tag" -> TagRoutes(restApp.backend, token),

View File

@ -13,8 +13,15 @@ case class CookieData(auth: AuthToken) {
def asCookie(cfg: Config): ResponseCookie = {
val domain = cfg.baseUrl.host
val sec = cfg.baseUrl.scheme.exists(_.endsWith("s"))
val path = cfg.baseUrl.path/"api"/"v1"/"sec"
ResponseCookie(CookieData.cookieName, asString, domain = domain, path = Some(path.asString), httpOnly = true, secure = sec)
val path = cfg.baseUrl.path / "api" / "v1" / "sec"
ResponseCookie(
CookieData.cookieName,
asString,
domain = domain,
path = Some(path.asString),
httpOnly = true,
secure = sec
)
}
}
object CookieData {
@ -22,18 +29,21 @@ object CookieData {
val headerName = "X-Docspell-Auth"
def authenticator[F[_]](r: Request[F]): Either[String, String] =
fromCookie(r) orElse fromHeader(r)
fromCookie(r).orElse(fromHeader(r))
def fromCookie[F[_]](req: Request[F]): Either[String, String] = {
def fromCookie[F[_]](req: Request[F]): Either[String, String] =
for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie")
cookie <- header.values.toList
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
} yield cookie.content
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] = {
req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator")
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] =
req.headers
.get(CaseInsensitiveString(headerName))
.map(_.value)
.toRight("Couldn't find an authenticator")
def deleteCookie(cfg: Config): ResponseCookie =
ResponseCookie(

View File

@ -24,31 +24,37 @@ trait Conversions {
// insights
def mkItemInsights(d: InsightData): ItemInsights =
ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2))))
ItemInsights(
d.incoming,
d.outgoing,
d.bytes,
TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))
)
// attachment meta
def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta =
AttachmentMeta(rm.content.getOrElse("")
, rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition))
, mkItemProposals(rm.proposals))
AttachmentMeta(
rm.content.getOrElse(""),
rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)),
mkItemProposals(rm.proposals)
)
// item proposal
def mkItemProposals(ml: MetaProposalList): ItemProposals = {
def get(mpt: MetaProposalType) =
ml.find(mpt).
map(mp => mp.values.toList.map(_.ref).map(mkIdName)).
getOrElse(Nil)
ml.find(mpt).map(mp => mp.values.toList.map(_.ref).map(mkIdName)).getOrElse(Nil)
def getDates(mpt: MetaProposalType): List[Timestamp] =
ml.find(mpt).
map(mp => mp.values.toList.
map(cand => cand.ref.id.id).
flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption).
map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))).
map(zdt => Timestamp(zdt.toInstant))).
getOrElse(Nil).
distinct.
take(5)
ml.find(mpt)
.map(mp =>
mp.values.toList
.map(cand => cand.ref.id.id)
.flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption)
.map(_.atTime(12, 0).atZone(ZoneId.of("GMT")))
.map(zdt => Timestamp(zdt.toInstant))
)
.getOrElse(Nil)
.distinct
.take(5)
ItemProposals(
corrOrg = get(MetaProposalType.CorrOrg),
@ -62,23 +68,25 @@ trait Conversions {
// item detail
def mkItemDetail(data: OItem.ItemData): ItemDetail =
ItemDetail(data.item.id
, data.item.direction
, data.item.name
, data.item.source
, data.item.state
, data.item.created
, data.item.updated
, data.item.itemDate
, data.corrOrg.map(o => IdName(o.oid, o.name))
, data.corrPerson.map(p => IdName(p.pid, p.name))
, data.concPerson.map(p => IdName(p.pid, p.name))
, data.concEquip.map(e => IdName(e.eid, e.name))
, data.inReplyTo.map(mkIdName)
, data.item.dueDate
, data.item.notes
, data.attachments.map((mkAttachment _).tupled).toList
, data.tags.map(mkTag).toList)
ItemDetail(
data.item.id,
data.item.direction,
data.item.name,
data.item.source,
data.item.state,
data.item.created,
data.item.updated,
data.item.itemDate,
data.corrOrg.map(o => IdName(o.oid, o.name)),
data.corrPerson.map(p => IdName(p.pid, p.name)),
data.concPerson.map(p => IdName(p.pid, p.name)),
data.concEquip.map(e => IdName(e.eid, e.name)),
data.inReplyTo.map(mkIdName),
data.item.dueDate,
data.item.notes,
data.attachments.map((mkAttachment _).tupled).toList,
data.tags.map(mkTag).toList
)
def mkAttachment(ra: RAttachment, m: FileMeta): Attachment =
Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString))
@ -86,20 +94,21 @@ trait Conversions {
// item list
def mkQuery(m: ItemSearch, coll: Ident): OItem.Query =
OItem.Query(coll
, m.name
, if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed)
, m.direction
, m.corrPerson
, m.corrOrg
, m.concPerson
, m.concEquip
, m.tagsInclude.map(Ident.unsafe)
, m.tagsExclude.map(Ident.unsafe)
, m.dateFrom
, m.dateUntil
, m.dueDateFrom
, m.dueDateUntil
OItem.Query(
coll,
m.name,
if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed),
m.direction,
m.corrPerson,
m.corrOrg,
m.concPerson,
m.concEquip,
m.tagsInclude.map(Ident.unsafe),
m.tagsExclude.map(Ident.unsafe),
m.dateFrom,
m.dateUntil,
m.dueDateFrom,
m.dueDateUntil
)
def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = {
@ -113,8 +122,20 @@ trait Conversions {
}
def mkItemLight(i: OItem.ListItem): ItemLight =
ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount)
ItemLight(
i.id,
i.name,
i.state,
i.date,
i.dueDate,
i.source,
i.direction.name.some,
i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName),
i.concPerson.map(mkIdName),
i.concEquip.map(mkIdName),
i.fileCount
)
// job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
@ -128,46 +149,57 @@ trait Conversions {
val t2 = f(j2).getOrElse(Timestamp.Epoch)
t1.value.isBefore(t2.value)
}
JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started))
, state.done.map(mkJobDetail).toList.sortWith(desc(_.finished))
, state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some)))
JobQueueState(
state.running.map(mkJobDetail).toList.sortWith(asc(_.started)),
state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)),
state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))
)
}
def mkJobDetail(jd: OJob.JobDetail): JobDetail =
JobDetail(jd.job.id
, jd.job.subject
, jd.job.submitted
, jd.job.priority
, jd.job.state
, jd.job.retries
, jd.logs.map(mkJobLog).toList
, jd.job.progress
, jd.job.worker
, jd.job.started
, jd.job.finished)
JobDetail(
jd.job.id,
jd.job.subject,
jd.job.submitted,
jd.job.priority,
jd.job.state,
jd.job.retries,
jd.logs.map(mkJobLog).toList,
jd.job.progress,
jd.job.worker,
jd.job.started,
jd.job.finished
)
def mkJobLog(jl: RJobLog): JobLogEvent =
JobLogEvent(jl.created, jl.level, jl.message)
// upload
def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = {
body.through(fs2.text.utf8Decode).
parseJsonAs[ItemUploadMeta].
map(_.fold(ex => {
def readMultipart[F[_]: Effect](
mp: Multipart[F],
logger: Logger,
prio: Priority,
validFileTypes: Seq[MimeType]
): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] =
body
.through(fs2.text.utf8Decode)
.parseJsonAs[ItemUploadMeta]
.map(_.fold(ex => {
logger.error(ex)("Reading upload metadata failed.")
throw ex
}, identity))
}
val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")).
map(p => parseMeta(p.body)).
map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))).
getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val meta: F[(Boolean, UploadMeta)] = mp.parts
.find(_.name.exists(_.equalsIgnoreCase("meta")))
.map(p => parseMeta(p.body))
.map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes))))
.getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val files = mp.parts.
filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))).
map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body))
val files = mp.parts
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
.map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
)
for {
metaData <- meta
_ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData"))
@ -178,8 +210,14 @@ trait Conversions {
// organization and person
def mkOrg(v: OOrganization.OrgAndContacts): Organization = {
val ro = v.org
Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList, ro.notes, ro.created)
Organization(
ro.oid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.created
)
}
def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = {
@ -189,7 +227,17 @@ trait Conversions {
now <- Timestamp.current[F]
oid <- Ident.randomId[F]
cont <- contacts(oid)
org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now)
org = ROrganization(
oid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
now
)
} yield OOrganization.OrgAndContacts(org, cont)
}
@ -198,14 +246,31 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, oid.some, None))
for {
cont <- contacts(v.id)
org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created)
org = ROrganization(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.created
)
} yield OOrganization.OrgAndContacts(org, cont)
}
def mkPerson(v: OOrganization.PersonAndContacts): Person = {
val ro = v.person
Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created)
Person(
ro.pid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.concerning,
ro.created
)
}
def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = {
@ -215,7 +280,18 @@ trait Conversions {
now <- Timestamp.current[F]
pid <- Ident.randomId[F]
cont <- contacts(pid)
org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now)
org = RPerson(
pid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
now
)
} yield OOrganization.PersonAndContacts(org, cont)
}
@ -224,7 +300,18 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, None, pid.some))
for {
cont <- contacts(v.id)
org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created)
org = RPerson(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
v.created
)
} yield OOrganization.PersonAndContacts(org, cont)
}
@ -233,7 +320,8 @@ trait Conversions {
Contact(rc.contactId, rc.value, rc.kind)
def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] =
timeId.map { case (id, now) =>
timeId.map {
case (id, now) =>
RContact(id, c.value, c.kind, pid, oid, now)
}
@ -242,12 +330,33 @@ trait Conversions {
User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created)
def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] =
timeId.map { case (id, now) =>
RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, 0, None, now)
timeId.map {
case (id, now) =>
RUser(
id,
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
0,
None,
now
)
}
def changeUser(u: User, cid: Ident): RUser =
RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created)
RUser(
Ident.unsafe(""),
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
u.loginCount,
u.lastLogin,
u.created
)
// tags
@ -255,21 +364,22 @@ trait Conversions {
Tag(rt.tagId, rt.name, rt.category, rt.created)
def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] =
timeId.map { case (id, now) =>
timeId.map {
case (id, now) =>
RTag(id, cid, t.name, t.category, now)
}
def changeTag(t: Tag, cid: Ident): RTag =
RTag(t.id, cid, t.name, t.category, t.created)
// sources
def mkSource(s: RSource): Source =
Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map({ case (id, now) =>
timeId.map({
case (id, now) =>
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
})
@ -281,7 +391,8 @@ trait Conversions {
Equipment(re.eid, re.name, re.created)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map({ case (id, now) =>
timeId.map({
case (id, now) =>
REquipment(id, cid, e.name, now)
})
@ -298,7 +409,8 @@ trait Conversions {
def basicResult(cr: JobCancelResult): BasicResult =
cr match {
case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.CancelRequested =>
BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.")
}
@ -317,7 +429,8 @@ trait Conversions {
def basicResult(cr: PassChangeResult): BasicResult = cr match {
case PassChangeResult.Success => BasicResult(true, "Password changed.")
case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.")
case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.")
case PassChangeResult.PasswordMismatch =>
BasicResult(false, "The current password is incorrect.")
case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
}

View File

@ -8,28 +8,26 @@ import org.http4s.dsl.Http4sDsl
trait ResponseGenerator[F[_]] {
self: Http4sDsl[F] =>
implicit final class EitherResponses[A,B](e: Either[A, B]) {
def toResponse(headers: Header*)
(implicit F: Applicative[F]
, w0: EntityEncoder[F, A]
, w1: EntityEncoder[F, B]): F[Response[F]] =
implicit final class EitherResponses[A, B](e: Either[A, B]) {
def toResponse(headers: Header*)(
implicit F: Applicative[F],
w0: EntityEncoder[F, A],
w1: EntityEncoder[F, B]
): F[Response[F]] =
e.fold(
a => UnprocessableEntity(a),
b => Ok(b)
).map(_.withHeaders(headers: _*))
)
.map(_.withHeaders(headers: _*))
}
implicit final class OptionResponse[A](o: Option[A]) {
def toResponse(headers: Header*)
(implicit F: Applicative[F]
, w0: EntityEncoder[F, A]): F[Response[F]] =
def toResponse(
headers: Header*
)(implicit F: Applicative[F], w0: EntityEncoder[F, A]): F[Response[F]] =
o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*))
}
}
object ResponseGenerator {
}
object ResponseGenerator {}

View File

@ -18,17 +18,18 @@ import org.http4s.headers.ETag.EntityTag
object AttachmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = {
val mt = MediaType.unsafeParse(data.meta.mimetype.asString)
val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length)
val eTag: Header = ETag(data.meta.checksum)
val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
Ok(data.data.take(data.meta.length)).
map(r => r.withContentType(`Content-Type`(mt)).
withHeaders(cntLen, eTag, disp))
val disp: Header =
`Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
Ok(data.data.take(data.meta.length)).map(r =>
r.withContentType(`Content-Type`(mt)).withHeaders(cntLen, eTag, disp)
)
}
HttpRoutes.of {
@ -38,7 +39,8 @@ object AttachmentRoutes {
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData, inm)
resp <- if (matches) NotModified()
else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
else
fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
case GET -> Root / Ident(id) / "meta" =>
@ -50,8 +52,10 @@ object AttachmentRoutes {
}
}
private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]]
, noneMatch: Option[NonEmptyList[EntityTag]]): Boolean =
private def matchETag[F[_]](
fileData: Option[OItem.AttachmentData[F]],
noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean =
(fileData, noneMatch) match {
case (Some(fd), Some(nm)) =>
fd.meta.checksum == nm.head.tag

View File

@ -12,14 +12,17 @@ import org.http4s.server._
object Authenticate {
def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] =
def authenticateRequest[F[_]: Effect](
auth: String => F[Login.Result]
)(req: Request[F]): F[Login.Result] =
CookieData.authenticator(req) match {
case Right(str) => auth(str)
case Left(_) => Login.Result.invalidAuth.pure[F]
}
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = {
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(
pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._
@ -34,7 +37,9 @@ object Authenticate {
middleware(AuthedRoutes.of(pf))
}
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = {
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(
f: AuthToken => HttpRoutes[F]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._
@ -49,6 +54,8 @@ object Authenticate {
middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req)))
}
private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] =
private def getUser[F[_]: Effect](
auth: String => F[Login.Result]
): Kleisli[F, Request[F], Either[String, AuthToken]] =
Kleisli(r => authenticateRequest(auth)(r).map(_.toEither))
}

View File

@ -25,7 +25,7 @@ object CollectiveRoutes {
resp <- Ok(Conversions.mkItemInsights(ins))
} yield resp
case req@POST -> Root / "settings" =>
case req @ POST -> Root / "settings" =>
for {
settings <- req.as[CollectiveSettings]
res <- backend.collective.updateLanguage(user.account.collective, settings.language)

View File

@ -15,7 +15,7 @@ import org.http4s.dsl.Http4sDsl
object EquipmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of {

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of[F] {
case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version
, BuildInfo.builtAtMillis
, BuildInfo.builtAtString
, BuildInfo.gitHeadCommit.getOrElse("")
, BuildInfo.gitDescribedVersion.getOrElse("")))
Ok(
VersionInfo(
BuildInfo.version,
BuildInfo.builtAtMillis,
BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
}
}
}

View File

@ -18,7 +18,7 @@ object ItemRoutes {
private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of {
@ -51,63 +51,63 @@ object ItemRoutes {
resp <- Ok(Conversions.basicResult(res, "Item back to created."))
} yield resp
case req@POST -> Root / Ident(id) / "tags" =>
case req @ POST -> Root / Ident(id) / "tags" =>
for {
tags <- req.as[ReferenceList].map(_.items)
res <- backend.item.setTags(id, tags.map(_.id), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Tags updated"))
} yield resp
case req@POST -> Root / Ident(id) / "direction" =>
case req @ POST -> Root / Ident(id) / "direction" =>
for {
dir <- req.as[DirectionValue]
res <- backend.item.setDirection(id, dir.direction, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Direction updated"))
} yield resp
case req@POST -> Root / Ident(id) / "corrOrg" =>
case req @ POST -> Root / Ident(id) / "corrOrg" =>
for {
idref <- req.as[OptionalId]
res <- backend.item.setCorrOrg(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
} yield resp
case req@POST -> Root / Ident(id) / "corrPerson" =>
case req @ POST -> Root / Ident(id) / "corrPerson" =>
for {
idref <- req.as[OptionalId]
res <- backend.item.setCorrPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
} yield resp
case req@POST -> Root / Ident(id) / "concPerson" =>
case req @ POST -> Root / Ident(id) / "concPerson" =>
for {
idref <- req.as[OptionalId]
res <- backend.item.setConcPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
} yield resp
case req@POST -> Root / Ident(id) / "concEquipment" =>
case req @ POST -> Root / Ident(id) / "concEquipment" =>
for {
idref <- req.as[OptionalId]
res <- backend.item.setConcEquip(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp
case req@POST -> Root / Ident(id) / "notes" =>
case req @ POST -> Root / Ident(id) / "notes" =>
for {
text <- req.as[OptionalText]
res <- backend.item.setNotes(id, text.text, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp
case req@POST -> Root / Ident(id) / "name" =>
case req @ POST -> Root / Ident(id) / "name" =>
for {
text <- req.as[OptionalText]
res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp
case req@POST -> Root / Ident(id) / "duedate" =>
case req @ POST -> Root / Ident(id) / "duedate" =>
for {
date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item due date to ${date.date}")
@ -115,7 +115,7 @@ object ItemRoutes {
resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
} yield resp
case req@POST -> Root / Ident(id) / "date" =>
case req @ POST -> Root / Ident(id) / "date" =>
for {
date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item date to ${date.date}")

View File

@ -18,7 +18,7 @@ object LoginRoutes {
import dsl._
HttpRoutes.of[F] {
case req@POST -> Root / "login" =>
case req @ POST -> Root / "login" =>
for {
up <- req.as[UserPass]
res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password))
@ -33,22 +33,36 @@ object LoginRoutes {
HttpRoutes.of[F] {
case req @ POST -> Root / "session" =>
Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req).
flatMap(res => makeResponse(dsl, cfg, res, ""))
Authenticate
.authenticateRequest(S.loginSession(cfg.auth))(req)
.flatMap(res => makeResponse(dsl, cfg, res, ""))
case POST -> Root / "logout" =>
Ok().map(_.addCookie(CookieData.deleteCookie(cfg)))
}
}
def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = {
def makeResponse[F[_]: Effect](
dsl: Http4sDsl[F],
cfg: Config,
res: Login.Result,
account: String
): F[Response[F]] = {
import dsl._
res match {
case Login.Result.Ok(token) =>
for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)).
map(_.addCookie(cd.asCookie(cfg)))
resp <- Ok(
AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
} yield resp
case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L))

View File

@ -16,7 +16,7 @@ import org.http4s.dsl.Http4sDsl
object OrganizationRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of {

View File

@ -6,9 +6,10 @@ import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher
object ParamDecoder {
implicit val booleanDecoder: QueryParamDecoder[Boolean] =
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean")
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_.equalsIgnoreCase("true")))(
"Boolean"
)
object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full")
}

View File

@ -19,7 +19,7 @@ object PersonRoutes {
private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of {

View File

@ -29,7 +29,7 @@ object RegisterRoutes {
resp <- Ok(convert(res))
} yield resp
case req@ POST -> Root / "newinvite" =>
case req @ POST -> Root / "newinvite" =>
for {
data <- req.as[GenInvite]
res <- backend.signup.newInvite(cfg.backend.signup)(data.password)
@ -47,7 +47,6 @@ object RegisterRoutes {
InviteResult(false, "Password is invalid.", None)
}
def convert(r: SignupResult): BasicResult = r match {
case SignupResult.CollectiveExists =>
BasicResult(false, "A collective with this name already exists.")
@ -62,7 +61,6 @@ object RegisterRoutes {
BasicResult(true, "Signup successful")
}
def convert(r: Registration): RegisterData =
RegisterData(r.collectiveName, r.login, r.password, r.invite)
}

View File

@ -26,7 +26,12 @@ object UploadRoutes {
case req @ POST -> Root / "item" =>
for {
multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes)
updata <- readMultipart(
multipart,
logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result))
} yield res
@ -39,7 +44,7 @@ object UploadRoutes {
import dsl._
HttpRoutes.of {
case req @ POST -> Root / "item" / Ident(id)=>
case req @ POST -> Root / "item" / Ident(id) =>
for {
multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes)

View File

@ -23,7 +23,11 @@ object UserRoutes {
case req @ POST -> Root / "changePassword" =>
for {
data <- req.as[PasswordChange]
res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword)
res <- backend.collective.changePassword(
user.account,
data.currentPassword,
data.newPassword
)
resp <- Ok(basicResult(res))
} yield resp

View File

@ -8,14 +8,21 @@ import docspell.backend.signup.{Config => SignupConfig}
import yamusca.imports._
import yamusca.implicits._
case class Flags( appName: String
, baseUrl: LenientUri
, signupMode: SignupConfig.Mode
, docspellAssetPath: String)
case class Flags(
appName: String,
baseUrl: LenientUri,
signupMode: SignupConfig.Mode,
docspellAssetPath: String
)
object Flags {
def apply(cfg: Config): Flags =
Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode, s"assets/docspell-webapp/${BuildInfo.version}")
Flags(
cfg.appName,
cfg.baseUrl,
cfg.backend.signup.mode,
s"assets/docspell-webapp/${BuildInfo.version}"
)
implicit val jsonEncoder: Encoder[Flags] =
deriveEncoder[Flags]

View File

@ -21,11 +21,13 @@ object TemplateRoutes {
val `text/html` = new MediaType("text", "html")
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit C: ContextShift[F]): HttpRoutes[F] = {
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(
implicit C: ContextShift[F]
): HttpRoutes[F] = {
val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker)))
val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker)))
val dsl = new Http4sDsl[F]{}
val dsl = new Http4sDsl[F] {}
import dsl._
HttpRoutes.of[F] {
case GET -> Root / "index.html" =>
@ -41,20 +43,21 @@ object TemplateRoutes {
}
}
def loadResource[F[_]: Sync](name: String): F[URL] = {
def loadResource[F[_]: Sync](name: String): F[URL] =
Option(getClass.getResource(name)) match {
case None =>
Sync[F].raiseError(new Exception("Unknown resource: "+ name))
Sync[F].raiseError(new Exception("Unknown resource: " + name))
case Some(r) =>
r.pure[F]
}
}
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] =
Stream.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())).
flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)).
through(text.utf8Decode).
compile.fold("")(_ + _)
Stream
.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
.flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false))
.through(text.utf8Decode)
.compile
.fold("")(_ + _)
def parseTemplate[F[_]: Sync](str: String): F[Template] =
Sync[F].delay {
@ -64,47 +67,54 @@ object TemplateRoutes {
}
}
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[Template] = {
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).
map(t => {
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(
implicit C: ContextShift[F]
): F[Template] =
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
logger.info(s"Compiled template $url")
t
})
}
case class DocData(swaggerRoot: String, openapiSpec: String)
object DocData {
def apply(): DocData =
DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml")
DocData(
"/app/assets" + Webjars.swaggerui,
s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml"
)
implicit def yamuscaValueConverter: ValueConverter[DocData] =
ValueConverter.deriveConverter[DocData]
}
case class IndexData(flags: Flags
, cssUrls: Seq[String]
, jsUrls: Seq[String]
, faviconBase: String
, appExtraJs: String
, flagsJson: String)
case class IndexData(
flags: Flags,
cssUrls: Seq[String],
jsUrls: Seq[String],
faviconBase: String,
appExtraJs: String,
flagsJson: String
)
object IndexData {
def apply(cfg: Config): IndexData =
IndexData(Flags(cfg)
, Seq(
IndexData(
Flags(cfg),
Seq(
"/app/assets" + Webjars.semanticui + "/semantic.min.css",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css"
)
, Seq(
),
Seq(
"/app/assets" + Webjars.jquery + "/jquery.min.js",
"/app/assets" + Webjars.semanticui + "/semantic.min.js",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js"
),
s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js",
Flags(cfg).asJson.spaces2
)
, s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon"
, s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js"
, Flags(cfg).asJson.spaces2 )
implicit def yamuscaValueConverter: ValueConverter[IndexData] =
ValueConverter.deriveConverter[IndexData]
@ -116,10 +126,10 @@ object TemplateRoutes {
Option(ref.get) match {
case Some(a) => a.pure[F]
case None =>
fa.map(a => {
fa.map { a =>
ref.set(a)
a
})
}
}
}
}

View File

@ -9,7 +9,7 @@ import org.http4s.server.staticcontent.WebjarService.{WebjarAsset, Config => Web
object WebjarRoutes {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] =
webjarService(
WebjarConfig(
filter = assetFilter,
@ -17,10 +17,23 @@ object WebjarRoutes {
cacheStrategy = NoopCacheStrategy[F]
)
)
}
def assetFilter(asset: WebjarAsset): Boolean =
List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml").
exists(e => asset.asset.endsWith(e))
List(
".js",
".css",
".html",
".json",
".jpg",
".png",
".eot",
".woff",
".woff2",
".svg",
".otf",
".ttf",
".yml",
".xml"
).exists(e => asset.asset.endsWith(e))
}

View File

@ -2,10 +2,7 @@ package docspell.store
import docspell.common.LenientUri
case class JdbcConfig(url: LenientUri
, user: String
, password: String
) {
case class JdbcConfig(url: LenientUri, user: String, password: String) {
val dbmsName: Option[String] =
JdbcConfig.extractDbmsName(url)

View File

@ -22,16 +22,20 @@ trait Store[F[_]] {
object Store {
def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig
, connectEC: ExecutionContext
, blocker: Blocker): Resource[F, Store[F]] = {
def create[F[_]: Effect: ContextShift](
jdbc: JdbcConfig,
connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, Store[F]] = {
val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass
, jdbc.url.asString
, jdbc.user
, jdbc.password
, connectEC
, blocker)
val hxa = HikariTransactor.newHikariTransactor[F](
jdbc.driverClass,
jdbc.url.asString,
jdbc.user,
jdbc.password,
connectEC,
blocker
)
for {
xa <- hxa

View File

@ -21,7 +21,9 @@ trait DoobieMeta {
})
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces)
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
e.apply(a).noSpaces
)
implicit val metaCollectiveState: Meta[CollectiveState] =
Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString)
@ -45,7 +47,9 @@ trait DoobieMeta {
Meta[String].imap(JobState.unsafe)(_.name)
implicit val metaDirection: Meta[Direction] =
Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d))
Meta[Boolean].imap(flag =>
if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction
)(d => Direction.isIncoming(d))
implicit val metaPriority: Meta[Priority] =
Meta[Int].imap(Priority.fromInt)(Priority.toInt)

View File

@ -19,7 +19,9 @@ trait DoobieSyntax {
commas(fa :: fas.toList)
def and(fs: Seq[Fragment]): Fragment =
Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
Fragment.const(" (") ++ fs
.filter(f => !isEmpty(f))
.reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
def and(f0: Fragment, fs: Fragment*): Fragment =
and(f0 :: fs.toList)
@ -48,8 +50,9 @@ trait DoobieSyntax {
def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment =
Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")"))
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(
vals.map(f => sql"(" ++ f ++ sql")")
)
def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment =
selectSimple(commas(cols.map(_.f)), table, where)
@ -62,7 +65,6 @@ trait DoobieSyntax {
Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++
Fragment.const(") FROM ") ++ table ++ this.where(where)
// def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment =
// selectSimple(cols.map(_.prefix("a"))
// , table ++ fr"a," ++ RCollective.table ++ fr"b"
@ -70,11 +72,12 @@ trait DoobieSyntax {
// else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b")))
def selectCount(col: Column, table: Fragment, where: Fragment): Fragment =
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where)
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(
where
)
def deleteFrom(table: Fragment, where: Fragment): Fragment = {
def deleteFrom(table: Fragment, where: Fragment): Fragment =
fr"DELETE FROM" ++ table ++ this.where(where)
}
def withCTE(ps: (String, Fragment)*): Fragment = {
val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")")

View File

@ -1,5 +1,3 @@
package docspell.store.impl
object Implicits extends DoobieMeta
with DoobieSyntax
object Implicits extends DoobieMeta with DoobieSyntax

View File

@ -10,7 +10,8 @@ import doobie._
import doobie.implicits._
final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] {
val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
val bitpeaceCfg =
BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
def migrate: F[Int] =
FlywayMigrate.run[F](jdbc)
@ -24,14 +25,14 @@ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends
def bitpeace: Bitpeace[F] =
Bitpeace(bitpeaceCfg, xa)
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = {
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] =
for {
save <- transact(insert).attempt
exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b)))
} yield exist.swap match {
case Right(_) => AddResult.Success
case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((_, true)) =>
AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((ex, _)) => AddResult.Failure(ex)
}
}
}

View File

@ -20,11 +20,12 @@ object FlywayMigrate {
}
logger.info(s"Using migration locations: $locations")
val fw = Flyway.configure().
cleanDisabled(true).
dataSource(jdbc.url.asString, jdbc.user, jdbc.password).
locations(locations: _*).
load()
val fw = Flyway
.configure()
.cleanDisabled(true)
.dataSource(jdbc.url.asString, jdbc.user, jdbc.password)
.locations(locations: _*)
.load()
fw.repair()
fw.migrate()

View File

@ -18,7 +18,7 @@ trait ONode[F[_]] {
object ONode {
private[this] val logger = getLogger
def apply[F[_] : Effect](store: Store[F]): Resource[F, ONode[F]] =
def apply[F[_]: Effect](store: Store[F]): Resource[F, ONode[F]] =
Resource.pure(new ONode[F] {
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =

View File

@ -12,40 +12,39 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for {
raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll))
n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId)))
f <- Stream.emit(raOpt).
unNoneTerminate.
map(_.fileId.id).
flatMap(store.bitpeace.delete).
compile.last
f <- Stream
.emit(raOpt)
.unNoneTerminate
.map(_.fileId.id)
.flatMap(store.bitpeace.delete)
.compile
.last
} yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0)
}
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = {
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for {
n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emit(ra.fileId.id).
flatMap(store.bitpeace.delete).
compile.last
f <- Stream.emit(ra.fileId.id).flatMap(store.bitpeace.delete).compile.last
} yield n + f.map(_ => 1).getOrElse(0)
}
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = {
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
for {
ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll))
ns <- ras.traverse(deleteAttachment[F](store))
} yield ns.sum
}
def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = {
val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns
val IC = RItem.Columns
val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
val q = fr"SELECT" ++ MC.proposals
.prefix("m")
.f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll))
@ -55,15 +54,22 @@ object QAttachment {
} yield MetaProposalList.flatten(ml)
}
def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = {
def getAttachmentMeta(
attachId: Ident,
collective: Ident
): ConnectionIO[Option[RAttachmentMeta]] = {
val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns
val IC = RItem.Columns
val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective)
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
.prefix("i")
.is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id
.prefix("a")
.is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a").is(attachId), IC.cid.prefix("i").is(collective))
q.query[RAttachmentMeta].option
}

View File

@ -8,27 +8,35 @@ import docspell.store.records.{RAttachment, RItem, RTag, RTagItem}
object QCollective {
case class InsightData( incoming: Int
, outgoing: Int
, bytes: Long
, tags: Map[String, Int])
case class InsightData(incoming: Int, outgoing: Int, bytes: Long, tags: Map[String, Int])
def getInsights(coll: Ident): ConnectionIO[InsightData] = {
val IC = RItem.Columns
val AC = RAttachment.Columns
val TC = RTag.Columns
val RC = RTagItem.Columns
val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)).
query[Int].unique
val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)).
query[Int].unique
val q0 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.incoming))
).query[Int].unique
val q1 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique
val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++
fr"WHERE" ++ IC.cid.is(coll)
val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++
val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
) ++
fr"FROM" ++ RTagItem.table ++ fr"r" ++
fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++
fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++

View File

@ -13,16 +13,18 @@ import docspell.store.impl.Implicits._
import org.log4s._
object QItem {
private [this] val logger = getLogger
private[this] val logger = getLogger
case class ItemData( item: RItem
, corrOrg: Option[ROrganization]
, corrPerson: Option[RPerson]
, concPerson: Option[RPerson]
, concEquip: Option[REquipment]
, inReplyTo: Option[IdRef]
, tags: Vector[RTag]
, attachments: Vector[(RAttachment, FileMeta)]) {
case class ItemData(
item: RItem,
corrOrg: Option[ROrganization],
corrPerson: Option[RPerson],
concPerson: Option[RPerson],
concEquip: Option[REquipment],
inReplyTo: Option[IdRef],
tags: Vector[RTag],
attachments: Vector[(RAttachment, FileMeta)]
) {
def filterCollective(coll: Ident): Option[ItemData] =
if (item.cid == coll) Some(this) else None
@ -37,14 +39,35 @@ object QItem {
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg.prefix("i").is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson.prefix("i").is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson.prefix("i").is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment.prefix("i").is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo.prefix("i").is(RItem.Columns.id.prefix("ref")) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
.prefix("i")
.is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.prefix("i")
.is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.prefix("i")
.is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val q = cq.query[(RItem, Option[ROrganization], Option[RPerson], Option[RPerson], Option[REquipment], Option[IdRef])].option
val q = cq
.query[
(
RItem,
Option[ROrganization],
Option[RPerson],
Option[RPerson],
Option[REquipment],
Option[IdRef]
)
]
.option
val attachs = RAttachment.findByItemWithMeta(id)
val tags = RTag.findByItem(id)
@ -56,35 +79,38 @@ object QItem {
} yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att))
}
case class ListItem(
id: Ident,
name: String,
state: ItemState,
date: Timestamp,
dueDate: Option[Timestamp],
source: String,
direction: Direction,
created: Timestamp,
fileCount: Int,
corrOrg: Option[IdRef],
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef]
)
case class ListItem( id: Ident
, name: String
, state: ItemState
, date: Timestamp
, dueDate: Option[Timestamp]
, source: String
, direction: Direction
, created: Timestamp
, fileCount: Int
, corrOrg: Option[IdRef]
, corrPerson: Option[IdRef]
, concPerson: Option[IdRef]
, concEquip: Option[IdRef])
case class Query( collective: Ident
, name: Option[String]
, states: Seq[ItemState]
, direction: Option[Direction]
, corrPerson: Option[Ident]
, corrOrg: Option[Ident]
, concPerson: Option[Ident]
, concEquip: Option[Ident]
, tagsInclude: List[Ident]
, tagsExclude: List[Ident]
, dateFrom: Option[Timestamp]
, dateTo: Option[Timestamp]
, dueDateFrom: Option[Timestamp]
, dueDateTo: Option[Timestamp])
case class Query(
collective: Ident,
name: Option[String],
states: Seq[ItemState],
direction: Option[Direction],
corrPerson: Option[Ident],
corrOrg: Option[Ident],
concPerson: Option[Ident],
concEquip: Option[Ident],
tagsInclude: List[Ident],
tagsExclude: List[Ident],
dateFrom: Option[Timestamp],
dateTo: Option[Timestamp],
dueDateFrom: Option[Timestamp],
dueDateTo: Option[Timestamp]
)
def findItems(q: Query): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns
@ -97,70 +123,93 @@ object QItem {
val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name)
val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name)
val finalCols = commas(IC.id.prefix("i").f
, IC.name.prefix("i").f
, IC.state.prefix("i").f
, coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f)
, IC.dueDate.prefix("i").f
, IC.source.prefix("i").f
, IC.incoming.prefix("i").f
, IC.created.prefix("i").f
, fr"COALESCE(a.num, 0)"
, OC.oid.prefix("o0").f
, OC.name.prefix("o0").f
, PC.pid.prefix("p0").f
, PC.name.prefix("p0").f
, PC.pid.prefix("p1").f
, PC.name.prefix("p1").f
, EC.eid.prefix("e1").f
, EC.name.prefix("e1").f
val finalCols = commas(
IC.id.prefix("i").f,
IC.name.prefix("i").f,
IC.state.prefix("i").f,
coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f),
IC.dueDate.prefix("i").f,
IC.source.prefix("i").f,
IC.incoming.prefix("i").f,
IC.created.prefix("i").f,
fr"COALESCE(a.num, 0)",
OC.oid.prefix("o0").f,
OC.name.prefix("o0").f,
PC.pid.prefix("p0").f,
PC.name.prefix("p0").f,
PC.pid.prefix("p1").f,
PC.name.prefix("p1").f,
EC.eid.prefix("e1").f,
EC.name.prefix("e1").f
)
val withItem = selectSimple(itemCols, RItem.table, IC.cid is q.collective)
val withPerson = selectSimple(personCols, RPerson.table, PC.cid is q.collective)
val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid is q.collective)
val withEquips = selectSimple(equipCols, REquipment.table, EC.cid is q.collective)
val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.collective))
val withPerson = selectSimple(personCols, RPerson.table, PC.cid.is(q.collective))
val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid.is(q.collective))
val withEquips = selectSimple(equipCols, REquipment.table, EC.cid.is(q.collective))
val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++
fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")"
val query = withCTE("items" -> withItem
, "persons" -> withPerson
, "orgs" -> withOrgs
, "equips" -> withEquips
, "attachs" -> withAttach) ++
val query = withCTE(
"items" -> withItem,
"persons" -> withPerson,
"orgs" -> withOrgs,
"equips" -> withEquips,
"attachs" -> withAttach
) ++
fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++
fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson
.prefix("i")
.is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg
.prefix("i")
.is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson
.prefix("i")
.is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
.prefix("i")
.is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude.map(tid =>
selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId is tid)).
map(f => sql"(" ++ f ++ sql") ")
val tagSelectsIncl = q.tagsInclude
.map(tid =>
selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId.is(tid))
)
.map(f => sql"(" ++ f ++ sql") ")
// exclusive tags are OR-ed
val tagSelectsExcl =
if (q.tagsExclude.isEmpty) Fragment.empty
else selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId isOneOf q.tagsExclude)
else
selectSimple(
List(RTagItem.Columns.itemId),
RTagItem.table,
RTagItem.Columns.tagId.isOneOf(q.tagsExclude)
)
val name = q.name.map(queryWildcard)
val cond = and(
IC.cid.prefix("i") is q.collective,
IC.state.prefix("i") isOneOf q.states,
IC.incoming.prefix("i") isOrDiscard q.direction,
IC.cid.prefix("i").is(q.collective),
IC.state.prefix("i").isOneOf(q.states),
IC.incoming.prefix("i").isOrDiscard(q.direction),
name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty),
RPerson.Columns.pid.prefix("p0") isOrDiscard q.corrPerson,
ROrganization.Columns.oid.prefix("o0") isOrDiscard q.corrOrg,
RPerson.Columns.pid.prefix("p1") isOrDiscard q.concPerson,
REquipment.Columns.eid.prefix("e1") isOrDiscard q.concEquip,
RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson),
ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg),
RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson),
REquipment.Columns.eid.prefix("e1").isOrDiscard(q.concEquip),
if (q.tagsInclude.isEmpty) Fragment.empty
else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
else
IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
if (q.tagsExclude.isEmpty) Fragment.empty
else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")",
q.dateFrom.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d").getOrElse(Fragment.empty),
q.dateTo.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d").getOrElse(Fragment.empty),
q.dateFrom
.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d")
.getOrElse(Fragment.empty),
q.dateTo
.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d")
.getOrElse(Fragment.empty),
q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty),
q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty)
)
@ -171,7 +220,6 @@ object QItem {
frag.query[ListItem].stream
}
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
for {
tn <- store.transact(RTagItem.deleteItemTags(itemId))
@ -183,7 +231,9 @@ object QItem {
val IC = RItem.Columns
val AC = RAttachment.Columns
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
q.query[RItem].to[Vector]

View File

@ -13,29 +13,38 @@ import fs2.Stream
import org.log4s._
object QJob {
private [this] val logger = getLogger
private[this] val logger = getLogger
def takeNextJob[F[_]: Effect](store: Store[F])(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = {
Stream.range(0, 10).
evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)).
evalTap({ x =>
if (x.isLeft) logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
def takeNextJob[F[_]: Effect](
store: Store[F]
)(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] =
Stream
.range(0, 10)
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
.evalTap({ x =>
if (x.isLeft)
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
else ().pure[F]
}).
find(_.isRight).
flatMap({
})
.find(_.isRight)
.flatMap({
case Right(job) =>
Stream.emit(job)
case Left(_) =>
Stream.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")).map(_ => None)
}).
compile.last.map(_.flatten)
}
Stream
.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
.map(_ => None)
})
.compile
.last
.map(_.flatten)
private def takeNextJob1[F[_]: Effect](store: Store[F])( priority: Ident => F[Priority]
, worker: Ident
, retryPause: Duration
, currentTry: Int): F[Either[Unit, Option[RJob]]] = {
private def takeNextJob1[F[_]: Effect](store: Store[F])(
priority: Ident => F[Priority],
worker: Ident,
retryPause: Duration,
currentTry: Int
): F[Either[Unit, Option[RJob]]] = {
//if this fails, we have to restart takeNextJob
def markJob(job: RJob): F[Either[Unit, RJob]] =
store.transact(for {
@ -51,7 +60,9 @@ object QJob {
_ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}")
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
job <- group.map(g => store.transact(selectNextJob(g, prio, retryPause, now))).getOrElse((None: Option[RJob]).pure[F])
job <- group
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F])
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
res <- job.traverse(j => markJob(j))
} yield res.map(_.map(_.some)).getOrElse {
@ -60,7 +71,11 @@ object QJob {
}
}
def selectNextGroup(worker: Ident, now: Timestamp, initialPause: Duration): ConnectionIO[Option[Ident]] = {
def selectNextGroup(
worker: Ident,
now: Timestamp,
initialPause: Duration
): ConnectionIO[Option[Ident]] = {
val JC = RJob.Columns
val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck
@ -72,21 +87,30 @@ object QJob {
val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++
fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}"
val stateCond = or(jstate is waiting, and(jstate is stuck, stuckTrigger ++ fr"< ${now.toMillis}"))
val stateCond =
or(jstate.is(waiting), and(jstate.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++
fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++
fr"WHERE" ++ and(uworker is worker, stateCond) ++
fr"WHERE" ++ and(uworker.is(worker), stateCond) ++
fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
fr"WHERE" ++ stateCond
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
union.query[Ident].to[List].map(_.headOption) // either one or two results, but may be empty if RJob table is empty
union
.query[Ident]
.to[List]
.map(_.headOption) // either one or two results, but may be empty if RJob table is empty
}
def selectNextJob(group: Ident, prio: Priority, initialPause: Duration, now: Timestamp): ConnectionIO[Option[RJob]] = {
def selectNextJob(
group: Ident,
prio: Priority,
initialPause: Duration,
now: Timestamp
): ConnectionIO[Option[RJob]] = {
val JC = RJob.Columns
val psort =
if (prio == Priority.High) JC.priority.desc
@ -94,9 +118,17 @@ object QJob {
val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck
val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries) ++ fr"* ${initialPause.millis}"
val sql = selectSimple(JC.all, RJob.table,
and(JC.group is group, or(JC.state is waiting, and(JC.state is stuck, stuckTrigger ++ fr"< ${now.toMillis}")))) ++
val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(
JC.retries
) ++ fr"* ${initialPause.millis}"
val sql = selectSimple(
JC.all,
RJob.table,
and(
JC.group.is(group),
or(JC.state.is(waiting), and(JC.state.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
)
) ++
orderBy(JC.state.asc, psort, JC.submitted.asc) ++
fr"LIMIT 1"
@ -150,9 +182,8 @@ object QJob {
def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] =
store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max))
def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = {
def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] =
store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ())
}
def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] =
store.transact(RJob.findFromIds(ids))
@ -165,10 +196,17 @@ object QJob {
def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = {
val refDate = now.minusHours(24)
val sql = selectSimple(JC.all, RJob.table,
and(JC.group is collective,
or(and(JC.state.isOneOf(done.toSeq), JC.submitted isGt refDate)
, JC.state.isOneOf((running ++ waiting).toSeq))))
val sql = selectSimple(
JC.all,
RJob.table,
and(
JC.group.is(collective),
or(
and(JC.state.isOneOf(done.toSeq), JC.submitted.isGt(refDate)),
JC.state.isOneOf((running ++ waiting).toSeq)
)
)
)
(sql ++ orderBy(JC.submitted.desc)).query[RJob].stream
}

View File

@ -12,10 +12,12 @@ import org.log4s._
object QLogin {
private[this] val logger = getLogger
case class Data( account: AccountId
, password: Password
, collectiveState: CollectiveState
, userState: UserState)
case class Data(
account: AccountId,
password: Password,
collectiveState: CollectiveState,
userState: UserState
)
def findUser(acc: AccountId): ConnectionIO[Option[Data]] = {
val ucid = UC.cid.prefix("u")
@ -26,9 +28,10 @@ object QLogin {
val ccid = CC.id.prefix("c")
val sql = selectSimple(
List(ucid,login,pass,cstate,ustate),
List(ucid, login, pass, cstate, ustate),
RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c",
and(ucid is ccid, login is acc.user, ucid is acc.collective))
and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective))
)
logger.trace(s"SQL : $sql")
sql.query[Data].option

View File

@ -12,16 +12,24 @@ import docspell.store.records._
object QOrganization {
def findOrgAndContact(coll: Ident, order: OC.type => Column): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = {
ROrganization.findAll(coll, order).
evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs)))
}
def findPersonAndContact(coll: Ident, order: PC.type => Column): Stream[ConnectionIO, (RPerson, Vector[RContact])] = {
RPerson.findAll(coll, order).
evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs)))
}
def findOrgAndContact(
coll: Ident,
order: OC.type => Column
): Stream[ConnectionIO, (ROrganization, Vector[RContact])] =
ROrganization
.findAll(coll, order)
.evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs)))
def findPersonAndContact(
coll: Ident,
order: PC.type => Column
): Stream[ConnectionIO, (RPerson, Vector[RContact])] =
RPerson.findAll(coll, order).evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs)))
def addOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def addOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- ROrganization.insert(org)
cs <- contacts.toList.traverse(RContact.insert)
@ -32,7 +40,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def addPerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def addPerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- RPerson.insert(person)
cs <- contacts.toList.traverse(RContact.insert)
@ -43,7 +55,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def updateOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def updateOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- ROrganization.update(org)
d <- RContact.deleteOrg(org.oid)
@ -55,7 +71,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def updatePerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def updatePerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- RPerson.update(person)
d <- RContact.deletePerson(person.pid)
@ -67,20 +87,18 @@ object QOrganization {
store => store.add(insert, exists)
}
def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = {
def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] =
for {
n0 <- RItem.removeCorrOrg(collective, orgId)
n1 <- RContact.deleteOrg(orgId)
n2 <- ROrganization.delete(orgId, collective)
} yield n0 + n1 + n2
}
def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = {
def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] =
for {
n0 <- RItem.removeCorrPerson(collective, personId)
n1 <- RItem.removeConcPerson(collective, personId)
n2 <- RContact.deletePerson(personId)
n3 <- RPerson.delete(personId, collective)
} yield n0 + n1 + n2 + n3
}
}

View File

@ -21,22 +21,29 @@ trait JobQueue[F[_]] {
object JobQueue {
private[this] val logger = getLogger
def apply[F[_] : Effect](store: Store[F]): Resource[F, JobQueue[F]] =
def apply[F[_]: Effect](store: Store[F]): Resource[F, JobQueue[F]] =
Resource.pure(new JobQueue[F] {
def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] =
def nextJob(
prio: Ident => F[Priority],
worker: Ident,
retryPause: Duration
): F[Option[RJob]] =
logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
def insert(job: RJob): F[Unit] =
store.transact(RJob.insert(job)).
flatMap({ n =>
if (n != 1) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
store
.transact(RJob.insert(job))
.flatMap({ n =>
if (n != 1)
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F]
})
def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList.traverse(j => insert(j).attempt).
map(_.foreach {
jobs.toList
.traverse(j => insert(j).attempt)
.map(_.foreach {
case Right(()) =>
case Left(ex) =>
logger.error(ex)("Could not insert job. Skipping it.")

View File

@ -7,14 +7,14 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RAttachment( id: Ident
, itemId: Ident
, fileId: Ident
, position: Int
, created: Timestamp
, name: Option[String]) {
}
case class RAttachment(
id: Ident,
itemId: Ident,
fileId: Ident,
position: Int,
created: Timestamp,
name: Option[String]
) {}
object RAttachment {
@ -32,25 +32,34 @@ object RAttachment {
import Columns._
def insert(v: RAttachment): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}").update.run
insertRow(
table,
all,
fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}"
).update.run
def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(all, table, id is attachId).query[RAttachment].option
selectSimple(all, table, id.is(attachId)).query[RAttachment].option
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = {
selectSimple(all.map(_.prefix("a")), table ++ fr"a," ++ RItem.table ++ fr"i", and(
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(
all.map(_.prefix("a")),
table ++ fr"a," ++ RItem.table ++ fr"i",
and(
fr"a.itemid = i.itemid",
id.prefix("a") is attachId,
RItem.Columns.cid.prefix("i") is collective
)).query[RAttachment].option
}
id.prefix("a").is(attachId),
RItem.Columns.cid.prefix("i").is(collective)
)
).query[RAttachment].option
def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] =
selectSimple(all, table, itemId is id).query[RAttachment].to[Vector]
selectSimple(all, table, itemId.is(id)).query[RAttachment].to[Vector]
def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = {
val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id.prefix("i").is(itemId.prefix("a")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id
.prefix("i")
.is(itemId.prefix("a")) ++
fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll))
q.query[RAttachment].to[Vector]
}
@ -65,7 +74,7 @@ object RAttachment {
def delete(attachId: Ident): ConnectionIO[Int] =
for {
n0 <- RAttachmentMeta.delete(attachId)
n1 <- deleteFrom(table, id is attachId).update.run
n1 <- deleteFrom(table, id.is(attachId)).update.run
} yield n0 + n1
}

View File

@ -7,12 +7,12 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RAttachmentMeta(id: Ident
, content: Option[String]
, nerlabels: List[NerLabel]
, proposals: MetaProposalList) {
}
case class RAttachmentMeta(
id: Ident,
content: Option[String],
nerlabels: List[NerLabel],
proposals: MetaProposalList
) {}
object RAttachmentMeta {
def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty)
@ -32,7 +32,7 @@ object RAttachmentMeta {
insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run
def exists(attachId: Ident): ConnectionIO[Boolean] =
selectCount(id, table, id is attachId).query[Int].unique.map(_ > 0)
selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0)
def upsert(v: RAttachmentMeta): ConnectionIO[Int] =
for {
@ -41,22 +41,34 @@ object RAttachmentMeta {
} yield n1
def update(v: RAttachmentMeta): ConnectionIO[Int] =
updateRow(table, id is v.id, commas(
content setTo v.content,
nerlabels setTo v.nerlabels,
proposals setTo v.proposals
)).update.run
updateRow(
table,
id.is(v.id),
commas(
content.setTo(v.content),
nerlabels.setTo(v.nerlabels),
proposals.setTo(v.proposals)
)
).update.run
def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] =
updateRow(table, id is mid, commas(
nerlabels setTo labels
)).update.run
updateRow(
table,
id.is(mid),
commas(
nerlabels.setTo(labels)
)
).update.run
def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] =
updateRow(table, id is mid, commas(
proposals setTo plist
)).update.run
updateRow(
table,
id.is(mid),
commas(
proposals.setTo(plist)
)
).update.run
def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id is attachId).update.run
deleteFrom(table, id.is(attachId)).update.run
}

View File

@ -7,10 +7,7 @@ import doobie._
import doobie.implicits._
import fs2.Stream
case class RCollective( id: Ident
, state: CollectiveState
, language: Language
, created: Timestamp)
case class RCollective(id: Ident, state: CollectiveState, language: Language, created: Timestamp)
object RCollective {
@ -29,30 +26,38 @@ object RCollective {
import Columns._
def insert(value: RCollective): ConnectionIO[Int] = {
val sql = insertRow(table, Columns.all, fr"${value.id},${value.state},${value.language},${value.created}")
val sql = insertRow(
table,
Columns.all,
fr"${value.id},${value.state},${value.language},${value.created}"
)
sql.update.run
}
def update(value: RCollective): ConnectionIO[Int] = {
val sql = updateRow(table, id is value.id, commas(
state setTo value.state
))
val sql = updateRow(
table,
id.is(value.id),
commas(
state.setTo(value.state)
)
)
sql.update.run
}
def findLanguage(cid: Ident): ConnectionIO[Option[Language]] =
selectSimple(List(language), table, id is cid).query[Option[Language]].unique
selectSimple(List(language), table, id.is(cid)).query[Option[Language]].unique
def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] =
updateRow(table, id is cid, language setTo lang).update.run
updateRow(table, id.is(cid), language.setTo(lang)).update.run
def findById(cid: Ident): ConnectionIO[Option[RCollective]] = {
val sql = selectSimple(all, table, id is cid)
val sql = selectSimple(all, table, id.is(cid))
sql.query[RCollective].option
}
def existsById(cid: Ident): ConnectionIO[Boolean] = {
val sql = selectCount(id, table, id is cid)
val sql = selectCount(id, table, id.is(cid))
sql.query[Int].unique.map(_ > 0)
}

View File

@ -6,14 +6,13 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RContact(
contactId: Ident
, value: String
, kind: ContactKind
, personId: Option[Ident]
, orgId: Option[Ident]
, created: Timestamp) {
}
contactId: Ident,
value: String,
kind: ContactKind,
personId: Option[Ident],
orgId: Option[Ident],
created: Timestamp
) {}
object RContact {
@ -26,48 +25,55 @@ object RContact {
val personId = Column("pid")
val orgId = Column("oid")
val created = Column("created")
val all = List(contactId, value,kind, personId, orgId, created)
val all = List(contactId, value, kind, personId, orgId, created)
}
import Columns._
def insert(v: RContact): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}")
val sql = insertRow(
table,
all,
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}"
)
sql.update.run
}
def update(v: RContact): ConnectionIO[Int] = {
val sql = updateRow(table, contactId is v.contactId, commas(
value setTo v.value,
kind setTo v.kind,
personId setTo v.personId,
orgId setTo v.orgId
))
val sql = updateRow(
table,
contactId.is(v.contactId),
commas(
value.setTo(v.value),
kind.setTo(v.kind),
personId.setTo(v.personId),
orgId.setTo(v.orgId)
)
)
sql.update.run
}
def delete(v: RContact): ConnectionIO[Int] =
deleteFrom(table, contactId is v.contactId).update.run
deleteFrom(table, contactId.is(v.contactId)).update.run
def deleteOrg(oid: Ident): ConnectionIO[Int] =
deleteFrom(table, orgId is oid).update.run
deleteFrom(table, orgId.is(oid)).update.run
def deletePerson(pid: Ident): ConnectionIO[Int] =
deleteFrom(table, personId is pid).update.run
deleteFrom(table, personId.is(pid)).update.run
def findById(id: Ident): ConnectionIO[Option[RContact]] = {
val sql = selectSimple(all, table, contactId is id)
val sql = selectSimple(all, table, contactId.is(id))
sql.query[RContact].option
}
def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, personId is pid)
val sql = selectSimple(all, table, personId.is(pid))
sql.query[RContact].to[Vector]
}
def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, orgId is oid)
val sql = selectSimple(all, table, orgId.is(oid))
sql.query[RContact].to[Vector]
}
}

View File

@ -5,13 +5,7 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class REquipment(
eid: Ident
, cid: Ident
, name: String
, created: Timestamp) {
}
case class REquipment(eid: Ident, cid: Ident, name: String, created: Timestamp) {}
object REquipment {
@ -22,44 +16,47 @@ object REquipment {
val cid = Column("cid")
val name = Column("name")
val created = Column("created")
val all = List(eid,cid,name,created)
val all = List(eid, cid, name, created)
}
import Columns._
def insert(v: REquipment): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.eid},${v.cid},${v.name},${v.created}")
val sql = insertRow(table, all, fr"${v.eid},${v.cid},${v.name},${v.created}")
sql.update.run
}
def update(v: REquipment): ConnectionIO[Int] = {
val sql = updateRow(table, and(eid is v.eid, cid is v.cid), commas(
cid setTo v.cid,
name setTo v.name
))
val sql = updateRow(
table,
and(eid.is(v.eid), cid.is(v.cid)),
commas(
cid.setTo(v.cid),
name.setTo(v.name)
)
)
sql.update.run
}
def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = {
val sql = selectCount(eid, table, and(cid is coll, name is ename))
val sql = selectCount(eid, table, and(cid.is(coll), name.is(ename)))
sql.query[Int].unique.map(_ > 0)
}
def findById(id: Ident): ConnectionIO[Option[REquipment]] = {
val sql = selectSimple(all, table, eid is id)
val sql = selectSimple(all, table, eid.is(id))
sql.query[REquipment].option
}
def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[REquipment].to[Vector]
}
def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] =
selectSimple(List(eid, name), table, and(cid is coll,
name.lowerLike(equipName))).
query[IdRef].to[Vector]
selectSimple(List(eid, name), table, and(cid.is(coll), name.lowerLike(equipName)))
.query[IdRef]
.to[Vector]
def delete(id: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(eid is id, cid is coll)).update.run
deleteFrom(table, and(eid.is(id), cid.is(coll))).update.run
}

View File

@ -8,9 +8,7 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RInvitation(id: Ident, created: Timestamp) {
}
case class RInvitation(id: Ident, created: Timestamp) {}
object RInvitation {
@ -33,18 +31,16 @@ object RInvitation {
insertRow(table, all, fr"${v.id},${v.created}").update.run
def insertNew: ConnectionIO[RInvitation] =
generate[ConnectionIO].
flatMap(v => insert(v).map(_ => v))
generate[ConnectionIO].flatMap(v => insert(v).map(_ => v))
def findById(invite: Ident): ConnectionIO[Option[RInvitation]] =
selectSimple(all, table, id is invite).query[RInvitation].option
selectSimple(all, table, id.is(invite)).query[RInvitation].option
def delete(invite: Ident): ConnectionIO[Int] =
deleteFrom(table, id is invite).update.run
deleteFrom(table, id.is(invite)).update.run
def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = {
val get = selectCount(id, table, and(id is invite, created isGt minCreated)).
query[Int].unique
val get = selectCount(id, table, and(id.is(invite), created.isGt(minCreated))).query[Int].unique
for {
inv <- get
_ <- delete(invite)

View File

@ -8,32 +8,55 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RItem( id: Ident
, cid: Ident
, name: String
, itemDate: Option[Timestamp]
, source: String
, direction: Direction
, state: ItemState
, corrOrg: Option[Ident]
, corrPerson: Option[Ident]
, concPerson: Option[Ident]
, concEquipment: Option[Ident]
, inReplyTo: Option[Ident]
, dueDate: Option[Timestamp]
, created: Timestamp
, updated: Timestamp
, notes: Option[String]) {
}
case class RItem(
id: Ident,
cid: Ident,
name: String,
itemDate: Option[Timestamp],
source: String,
direction: Direction,
state: ItemState,
corrOrg: Option[Ident],
corrPerson: Option[Ident],
concPerson: Option[Ident],
concEquipment: Option[Ident],
inReplyTo: Option[Ident],
dueDate: Option[Timestamp],
created: Timestamp,
updated: Timestamp,
notes: Option[String]
) {}
object RItem {
def newItem[F[_]: Sync](cid: Ident, name: String, source: String, direction: Direction, state: ItemState): F[RItem] =
def newItem[F[_]: Sync](
cid: Ident,
name: String,
source: String,
direction: Direction,
state: ItemState
): F[RItem] =
for {
now <- Timestamp.current[F]
id <- Ident.randomId[F]
} yield RItem(id, cid, name, None, source, direction, state, None, None, None, None, None, None, now, now, None)
} yield RItem(
id,
cid,
name,
None,
source,
direction,
state,
None,
None,
None,
None,
None,
None,
now,
now,
None
)
val table = fr"item"
@ -54,110 +77,189 @@ object RItem {
val created = Column("created")
val updated = Column("updated")
val notes = Column("notes")
val all = List(id, cid, name, itemDate, source, incoming, state, corrOrg,
corrPerson, concPerson, concEquipment, inReplyTo, dueDate, created, updated, notes)
val all = List(
id,
cid,
name,
itemDate,
source,
incoming,
state,
corrOrg,
corrPerson,
concPerson,
concEquipment,
inReplyTo,
dueDate,
created,
updated,
notes
)
}
import Columns._
def insert(v: RItem): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++
insertRow(
table,
all,
fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++
fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++
fr"${v.created},${v.updated},${v.notes}").update.run
fr"${v.created},${v.updated},${v.notes}"
).update.run
def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] =
selectSimple(List(cid), table, id is itemId).query[Ident].option
selectSimple(List(cid), table, id.is(itemId)).query[Ident].option
def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, id is itemId, commas(state setTo itemState, updated setTo t)).update.run
n <- updateRow(table, id.is(itemId), commas(state.setTo(itemState), updated.setTo(t))).update.run
} yield n
def updateStateForCollective(itemId: Ident, itemState: ItemState, coll: Ident): ConnectionIO[Int] =
def updateStateForCollective(
itemId: Ident,
itemState: ItemState,
coll: Ident
): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(state setTo itemState, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t))
).update.run
} yield n
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(incoming setTo dir, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t))
).update.run
} yield n
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrOrg setTo org, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t))
).update.run
} yield n
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, corrOrg is Some(currentOrg)), commas(corrOrg setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrPerson setTo person, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, corrPerson is Some(currentPerson)), commas(corrPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concPerson setTo person, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, concPerson is Some(currentPerson)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concEquipment setTo equip, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run
} yield n
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, concEquipment is Some(currentEquip)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(notes setTo text, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t))
).update.run
} yield n
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(name setTo itemName, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t))
).update.run
} yield n
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(itemDate setTo date, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(dueDate setTo date, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(id is itemId, cid is coll)).update.run
deleteFrom(table, and(id.is(itemId), cid.is(coll))).update.run
}

Some files were not shown because too many files have changed in this diff Show More