Apply scalafmt to all files

This commit is contained in:
Eike Kettner 2019-12-30 21:44:13 +01:00
parent 57e274e2b0
commit fc3e22e399
133 changed files with 3003 additions and 2112 deletions

View File

@ -27,7 +27,11 @@ trait BackendApp[F[_]] {
object BackendApp {
def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] =
def create[F[_]: ConcurrentEffect](
cfg: Config,
store: Store[F],
httpClientEc: ExecutionContext
): Resource[F, BackendApp[F]] =
for {
queue <- JobQueue(store)
loginImpl <- Login[F](store)
@ -55,10 +59,12 @@ object BackendApp {
val item = itemImpl
}
def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config
, connectEC: ExecutionContext
, httpClientEc: ExecutionContext
, blocker: Blocker): Resource[F, BackendApp[F]] =
def apply[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, BackendApp[F]] =
for {
store <- Store.create(cfg.jdbc, connectEC, blocker)
backend <- create(cfg, store, httpClientEc)

View File

@ -4,11 +4,7 @@ import docspell.backend.signup.{Config => SignupConfig}
import docspell.common.MimeType
import docspell.store.JdbcConfig
case class Config( jdbc: JdbcConfig
, signup: SignupConfig
, files: Config.Files) {
}
case class Config(jdbc: JdbcConfig, signup: SignupConfig, files: Config.Files) {}
object Config {

View File

@ -50,14 +50,13 @@ object AuthToken {
Left("Invalid authenticator")
}
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = {
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for {
salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key)
} yield cd.copy(sig = sig)
}
private def sign(cd: AuthToken, key: ByteVector): String = {
val raw = cd.millis.toString + cd.account.asString + cd.salt

View File

@ -49,7 +49,8 @@ object Login {
def invalidTime: Result = InvalidTime
}
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] {
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] =
Resource.pure(new Login[F] {
def loginSession(config: Config)(sessionKey: String): F[Result] =
AuthToken.fromString(sessionKey) match {
@ -61,7 +62,7 @@ object Login {
Result.invalidAuth.pure[F]
}
def loginUserPass(config: Config)(up: UserPass): F[Result] = {
def loginUserPass(config: Config)(up: UserPass): F[Result] =
AccountId.parse(up.user) match {
case Right(acc) =>
val okResult =
@ -76,7 +77,6 @@ object Login {
case Left(_) =>
Result.invalidAuth.pure[F]
}
}
private def check(given: String)(data: QLogin.Data): Boolean = {
val collOk = data.collectiveState == CollectiveState.Active ||

View File

@ -25,7 +25,11 @@ trait OCollective[F[_]] {
def insights(collective: Ident): F[InsightData]
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult]
def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult]
}
object OCollective {
@ -63,38 +67,46 @@ object OCollective {
}
}
def apply[F[_]: Effect](store: Store[F]): Resource[F, OCollective[F]] =
Resource.pure(new OCollective[F] {
def find(name: Ident): F[Option[RCollective]] =
store.transact(RCollective.findById(name))
def updateLanguage(collective: Ident, lang: Language): F[AddResult] =
store.transact(RCollective.updateLanguage(collective, lang)).
attempt.map(AddResult.fromUpdate)
store
.transact(RCollective.updateLanguage(collective, lang))
.attempt
.map(AddResult.fromUpdate)
def listUser(collective: Ident): F[Vector[RUser]] = {
def listUser(collective: Ident): F[Vector[RUser]] =
store.transact(RUser.findAll(collective, _.login))
}
def add(s: RUser): F[AddResult] =
store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login))
store.add(
RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))),
RUser.exists(s.login)
)
def update(s: RUser): F[AddResult] =
store.add(RUser.update(s), RUser.exists(s.login))
def deleteUser(login: Ident, collective: Ident): F[AddResult] =
store.transact(RUser.delete(login, collective)).
attempt.map(AddResult.fromUpdate)
store.transact(RUser.delete(login, collective)).attempt.map(AddResult.fromUpdate)
def insights(collective: Ident): F[InsightData] =
store.transact(QCollective.getInsights(collective))
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = {
def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult] = {
val q = for {
optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
n <- check
.filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match {
case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed

View File

@ -17,7 +17,6 @@ trait OEquipment[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult]
}
object OEquipment {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] =
@ -46,9 +45,7 @@ object OEquipment {
n0 <- RItem.removeConcEquip(collective, id)
n1 <- REquipment.delete(id, collective)
} yield n0 + n1
store.transact(io).
attempt.
map(AddResult.fromUpdate)
store.transact(io).attempt.map(AddResult.fromUpdate)
}
})
}

View File

@ -62,90 +62,98 @@ object OItem {
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] =
Resource.pure(new OItem[F] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store.transact(QItem.findItem(id)).
map(opt => opt.flatMap(_.filterCollective(collective)))
store.transact(QItem.findItem(id)).map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = {
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] =
store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector
}
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = {
store.transact(RAttachment.findByIdAndCollective(id, collective)).
flatMap({
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) =>
store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last.
map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m)))))
store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
)
)
case None =>
(None: Option[AttachmentData[F]]).pure[F]
})
}
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for {
cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO]
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
} yield nd + ni
store.transact(db).
attempt.
map(AddResult.fromUpdate)
store.transact(db).attempt.map(AddResult.fromUpdate)
}
def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] =
store.transact(RItem.updateDirection(item, collective, direction)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateDirection(item, collective, direction))
.attempt
.map(AddResult.fromUpdate)
def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrOrg(item, collective, org)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateCorrOrg(item, collective, org)).attempt.map(AddResult.fromUpdate)
def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrPerson(item, collective, person)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateCorrPerson(item, collective, person))
.attempt
.map(AddResult.fromUpdate)
def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcPerson(item, collective, person)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateConcPerson(item, collective, person))
.attempt
.map(AddResult.fromUpdate)
def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcEquip(item, collective, equip)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateConcEquip(item, collective, equip))
.attempt
.map(AddResult.fromUpdate)
def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] =
store.transact(RItem.updateNotes(item, collective, notes)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateNotes(item, collective, notes)).attempt.map(AddResult.fromUpdate)
def setName(item: Ident, name: String, collective: Ident): F[AddResult] =
store.transact(RItem.updateName(item, collective, name)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateName(item, collective, name)).attempt.map(AddResult.fromUpdate)
def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] =
store.transact(RItem.updateStateForCollective(item, state, collective)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateStateForCollective(item, state, collective))
.attempt
.map(AddResult.fromUpdate)
def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDate(item, collective, date)).
attempt.
map(AddResult.fromUpdate)
store.transact(RItem.updateDate(item, collective, date)).attempt.map(AddResult.fromUpdate)
def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDueDate(item, collective, date)).
attempt.
map(AddResult.fromUpdate)
store
.transact(RItem.updateDueDate(item, collective, date))
.attempt
.map(AddResult.fromUpdate)
def delete(itemId: Ident, collective: Ident): F[Int] =
QItem.delete(store)(itemId, collective)

View File

@ -36,15 +36,19 @@ object OJob {
jobs.filter(_.job.state == JobState.Running)
}
def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] =
def apply[F[_]: ConcurrentEffect](
store: Store[F],
clientEC: ExecutionContext
): Resource[F, OJob[F]] =
Resource.pure(new OJob[F] {
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = {
store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)).
map(t => JobDetail(t._1, t._2)).
compile.toVector.
map(CollectiveQueueState)
}
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
store
.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong))
.map(t => JobDetail(t._1, t._2))
.compile
.toVector
.map(CollectiveQueueState)
def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = {
def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] =
@ -66,8 +70,9 @@ object OJob {
}
def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] =
OJoex.cancelJob(job.id, worker, store, clientEC).
map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
OJoex
.cancelJob(job.id, worker, store, clientEC)
.map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
for {
tryDel <- store.transact(tryDelete)

View File

@ -15,21 +15,29 @@ import org.log4s._
object OJoex {
private[this] val logger = getLogger
def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = {
def notifyAll[F[_]: ConcurrentEffect](
store: Store[F],
clientExecutionContext: ExecutionContext
): F[Unit] =
for {
nodes <- store.transact(RNode.findAll(NodeType.Joex))
_ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext))
} yield ()
}
def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] =
def cancelJob[F[_]: ConcurrentEffect](
jobId: Ident,
worker: Ident,
store: Store[F],
clientEc: ExecutionContext
): F[Boolean] =
for {
node <- store.transact(RNode.findById(worker))
cancel <- node.traverse(joexCancel(clientEc)(_, jobId))
} yield cancel.getOrElse(false)
private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = {
private def joexCancel[F[_]: ConcurrentEffect](
ec: ExecutionContext
)(node: RNode, job: Ident): F[Boolean] = {
val notifyUrl = node.url / "api" / "v1" / "job" / job.id / "cancel"
BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))

View File

@ -40,9 +40,11 @@ object OOrganization {
Resource.pure(new OOrganization[F] {
def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] =
store.transact(QOrganization.findOrgAndContact(account.collective, _.name)).
map({ case (org, cont) => OrgAndContacts(org, cont) }).
compile.toVector
store
.transact(QOrganization.findOrgAndContact(account.collective, _.name))
.map({ case (org, cont) => OrgAndContacts(org, cont) })
.compile
.toVector
def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(ROrganization.findAllRef(account.collective, _.name))
@ -54,9 +56,11 @@ object OOrganization {
QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store)
def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] =
store.transact(QOrganization.findPersonAndContact(account.collective, _.name)).
map({ case (person, cont) => PersonAndContacts(person, cont) }).
compile.toVector
store
.transact(QOrganization.findPersonAndContact(account.collective, _.name))
.map({ case (person, cont) => PersonAndContacts(person, cont) })
.compile
.toVector
def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(RPerson.findAllRef(account.collective, _.name))
@ -68,14 +72,13 @@ object OOrganization {
QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store)
def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deleteOrg(orgId, collective)).
attempt.
map(AddResult.fromUpdate)
store.transact(QOrganization.deleteOrg(orgId, collective)).attempt.map(AddResult.fromUpdate)
def deletePerson(personId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deletePerson(personId, collective)).
attempt.
map(AddResult.fromUpdate)
store
.transact(QOrganization.deletePerson(personId, collective))
.attempt
.map(AddResult.fromUpdate)
})
}

View File

@ -41,8 +41,6 @@ object OSource {
}
def delete(id: Ident, collective: Ident): F[AddResult] =
store.transact(RSource.delete(id, collective)).
attempt.
map(AddResult.fromUpdate)
store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
})
}

View File

@ -17,7 +17,6 @@ trait OTag[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult]
}
object OTag {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] =
@ -47,10 +46,7 @@ object OTag {
n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective))
} yield n0.getOrElse(0) + n1.getOrElse(0)
store.transact(io).
attempt.
map(AddResult.fromUpdate)
store.transact(io).attempt.map(AddResult.fromUpdate)
}
})
}

View File

@ -24,15 +24,25 @@ trait OUpload[F[_]] {
object OUpload {
private[this] val logger = getLogger
case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte])
case class File[F[_]](
name: Option[String],
advertisedMime: Option[MimeType],
data: Stream[F, Byte]
)
case class UploadMeta( direction: Option[Direction]
, sourceAbbrev: String
, validFileTypes: Seq[MimeType])
case class UploadMeta(
direction: Option[Direction],
sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
case class UploadData[F[_]]( multiple: Boolean
, meta: UploadMeta
, files: Vector[File[F]], priority: Priority, tracker: Option[Ident])
case class UploadData[F[_]](
multiple: Boolean,
meta: UploadMeta,
files: Vector[File[F]],
priority: Priority,
tracker: Option[Ident]
)
sealed trait UploadResult
object UploadResult {
@ -41,22 +51,33 @@ object OUpload {
case object NoSource extends UploadResult
}
def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] =
def apply[F[_]: ConcurrentEffect](
store: Store[F],
queue: JobQueue[F],
cfg: Config,
httpClientEC: ExecutionContext
): Resource[F, OUpload[F]] =
Resource.pure(new OUpload[F] {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] =
for {
files <- data.files.traverse(saveFile).map(_.flatten)
pred <- checkFileList(files)
lang <- store.transact(RCollective.findLanguage(account.collective))
meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList))
meta = ProcessItemArgs.ProcessMeta(
account.collective,
lang.getOrElse(Language.German),
data.meta.direction,
data.meta.sourceAbbrev,
data.meta.validFileTypes
)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
else Vector(ProcessItemArgs(meta, files.toList))
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
_ <- logger.fdebug(s"Storing jobs: $job")
res <- job.traverse(submitJobs)
_ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective))
} yield res.fold(identity, identity)
}
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for {
@ -67,30 +88,47 @@ object OUpload {
result <- accId.traverse(acc => submit(updata, acc))
} yield result.fold(identity, identity)
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = {
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] =
for {
_ <- logger.fdebug(s"Storing jobs: $jobs")
_ <- queue.insertAll(jobs)
_ <- OJoex.notifyAll(store, httpClientEC)
} yield UploadResult.Success
}
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = {
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
logger.finfo(s"Receiving file $file") *>
store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None).
compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt.
map(_.fold(ex => {
store.bitpeace
.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None)
.compile
.lastOrError
.map(fm => Ident.unsafe(fm.id))
.attempt
.map(_.fold(ex => {
logger.warn(ex)(s"Could not store file for processing!")
None
}, id => Some(ProcessItemArgs.File(file.name, id))))
}
private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] =
Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(()))
private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = {
private def makeJobs(
args: Vector[ProcessItemArgs],
account: AccountId,
prio: Priority,
tracker: Option[Ident]
): F[Vector[RJob]] = {
def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob =
RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker)
RJob.newJob(
id,
ProcessItemArgs.taskName,
account.collective,
arg,
arg.makeSubject,
now,
account.user,
prio,
tracker
)
for {
id <- Ident.randomId[F]

View File

@ -24,16 +24,16 @@ object OSignup {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OSignup[F]] =
Resource.pure(new OSignup[F] {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
if (cfg.mode == Config.Mode.Invite) {
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F]
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
NewInviteResult.passwordMismatch.pure[F]
else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id))
} else {
Effect[F].pure(NewInviteResult.invitationClosed)
}
}
def register(cfg: Config)(data: RegisterData): F[SignupResult] = {
def register(cfg: Config)(data: RegisterData): F[SignupResult] =
cfg.mode match {
case Config.Mode.Open =>
addUser(data).map(SignupResult.fromAddResult)
@ -61,7 +61,6 @@ object OSignup {
SignupResult.invalidInvitationKey.pure[F]
}
}
}
private def retryInvite(res: SignupResult): Boolean =
res match {
@ -77,22 +76,30 @@ object OSignup {
false
}
private def addUser(data: RegisterData): F[AddResult] = {
def toRecords: F[(RCollective, RUser)] =
for {
id2 <- Ident.randomId[F]
now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now)
u = RUser(
id2,
data.login,
data.collName,
PasswordCrypt.crypt(data.password),
UserState.Active,
None,
0,
None,
now
)
} yield (c, u)
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = {
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] =
for {
n1 <- RCollective.insert(coll)
n2 <- RUser.insert(user)
} yield n1 + n2
}
def collectiveExists: ConnectionIO[Boolean] =
RCollective.existsById(data.collName)

View File

@ -2,9 +2,7 @@ package docspell.backend.signup
import docspell.store.AddResult
sealed trait SignupResult {
}
sealed trait SignupResult {}
object SignupResult {

View File

@ -1,12 +1,14 @@
package docspell.common
case class Banner( component: String
, version: String
, gitHash: Option[String]
, jdbcUrl: LenientUri
, configFile: Option[String]
, appId: Ident
, baseUrl: LenientUri) {
case class Banner(
component: String,
version: String,
gitHash: Option[String],
jdbcUrl: LenientUri,
configFile: Option[String],
appId: Ident,
baseUrl: LenientUri
) {
private val banner =
"""______ _ _
@ -21,12 +23,12 @@ case class Banner( component: String
def render(prefix: String): String = {
val text = banner.split('\n').toList ++ List(
s"<< $component >>"
, s"Id: ${appId.id}"
, s"Base-Url: ${baseUrl.asString}"
, s"Database: ${jdbcUrl.asString}"
, s"Config: ${configFile.getOrElse("")}"
, ""
s"<< $component >>",
s"Id: ${appId.id}",
s"Base-Url: ${baseUrl.asString}",
s"Database: ${jdbcUrl.asString}",
s"Config: ${configFile.getOrElse("")}",
""
)
text.map(line => s"$prefix $line").mkString("\n")

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe._
import io.circe.generic.semiauto._
case class IdRef(id: Ident, name: String) {
}
case class IdRef(id: Ident, name: String) {}
object IdRef {

View File

@ -32,4 +32,3 @@ object ItemState {
implicit val jsonEncoder: Encoder[ItemState] =
Encoder.encodeString.contramap(_.name)
}

View File

@ -31,14 +31,12 @@ object Language {
def fromString(str: String): Either[String, Language] = {
val lang = str.toLowerCase
all.find(_.allNames.contains(lang)).
toRight(s"Unsupported or invalid language: $str")
all.find(_.allNames.contains(lang)).toRight(s"Unsupported or invalid language: $str")
}
def unsafe(str: String): Language =
fromString(str).fold(sys.error, identity)
implicit val jsonDecoder: Decoder[Language] =
Decoder.decodeString.emap(fromString)
implicit val jsonEncoder: Encoder[Language] =

View File

@ -26,9 +26,10 @@ object MimeType {
def image(sub: String): MimeType =
MimeType("image", partFromString(sub).throwLeft)
private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
private[this] val validChars: Set[Char] =
(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
def parse(str: String): Either[String, MimeType] = {
def parse(str: String): Either[String, MimeType] =
str.indexOf('/') match {
case -1 => Left(s"Invalid MIME type: $str")
case n =>
@ -37,7 +38,6 @@ object MimeType {
sub <- partFromString(str.substring(n + 1))
} yield MimeType(prim.toLowerCase, sub.toLowerCase)
}
}
def unsafe(str: String): MimeType =
parse(str).throwLeft

View File

@ -2,6 +2,4 @@ package docspell.common
import java.time.LocalDate
case class NerDateLabel(date: LocalDate, label: NerLabel) {
}
case class NerDateLabel(date: LocalDate, label: NerLabel) {}

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {
}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {}
object NerLabel {
implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel]

View File

@ -16,7 +16,6 @@ object Priority {
case object Low extends Priority
def fromString(str: String): Either[String, Priority] =
str.toLowerCase match {
case "high" => Right(High)
@ -27,7 +26,6 @@ object Priority {
def unsafe(str: String): Priority =
fromString(str).fold(sys.error, identity)
def fromInt(n: Int): Priority =
if (n <= toInt(Low)) Low
else High

View File

@ -6,14 +6,13 @@ import ProcessItemArgs._
case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
def makeSubject: String = {
def makeSubject: String =
files.flatMap(_.name) match {
case Nil => s"${meta.sourceAbbrev}: No files"
case n :: Nil => n
case n1 :: n2 :: Nil => s"$n1, $n2"
case _ => s"${files.size} files from ${meta.sourceAbbrev}"
}
}
}
@ -21,11 +20,13 @@ object ProcessItemArgs {
val taskName = Ident.unsafe("process-item")
case class ProcessMeta( collective: Ident
, language: Language
, direction: Option[Direction]
, sourceAbbrev: String
, validFileTypes: Seq[MimeType])
case class ProcessMeta(
collective: Ident,
language: Language,
direction: Option[Direction],
sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
object ProcessMeta {
implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta]

View File

@ -30,8 +30,6 @@ object Timestamp {
def current[F[_]: Sync]: F[Timestamp] =
Sync[F].delay(Timestamp(Instant.now))
implicit val encodeTimestamp: Encoder[Timestamp] =
BaseJsonCodecs.encodeInstantEpoch.contramap(_.value)

View File

@ -12,7 +12,6 @@ object UserState {
/** The user is blocked by an admin. */
case object Disabled extends UserState
def fromString(s: String): Either[String, UserState] =
s.toLowerCase match {
case "active" => Right(Active)

View File

@ -11,13 +11,18 @@ trait StreamSyntax {
implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
s.fold("")(_ + _).
compile.last.
map(optStr => for {
str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value"))
s.fold("")(_ + _)
.compile
.last
.map(optStr =>
for {
str <- optStr
.map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying)
value <- json.as[A]
} yield value)
} yield value
)
}

View File

@ -5,17 +5,23 @@ import docspell.joex.scheduler.SchedulerConfig
import docspell.store.JdbcConfig
import docspell.text.ocr.{Config => OcrConfig}
case class Config(appId: Ident
, baseUrl: LenientUri
, bind: Config.Bind
, jdbc: JdbcConfig
, scheduler: SchedulerConfig
, extraction: OcrConfig
case class Config(
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
jdbc: JdbcConfig,
scheduler: SchedulerConfig,
extraction: OcrConfig
)
object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "")
val postgres =
JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
case class Bind(address: String, port: Int)
}

View File

@ -11,7 +11,6 @@ object ConfigFile {
def loadConfig: Config =
ConfigSource.default.at("docspell.joex").loadOrThrow[Config]
object Implicits {
implicit val countingSchemeReader: ConfigReader[CountingScheme] =
ConfigReader[String].emap(reason(CountingScheme.readString))

View File

@ -12,11 +12,13 @@ import fs2.concurrent.SignallingRef
import scala.concurrent.ExecutionContext
final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config
, nodeOps: ONode[F]
, store: Store[F]
, termSignal: SignallingRef[F, Boolean]
, val scheduler: Scheduler[F]) extends JoexApp[F] {
final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer](
cfg: Config,
nodeOps: ONode[F],
store: Store[F],
termSignal: SignallingRef[F, Boolean],
val scheduler: Scheduler[F]
) extends JoexApp[F] {
def init: F[Unit] = {
val run = scheduler.start.compile.drain
@ -40,16 +42,24 @@ final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Conf
object JoexAppImpl {
def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config
, termSignal: SignallingRef[F, Boolean]
, connectEC: ExecutionContext
, blocker: Blocker): Resource[F, JoexApp[F]] =
def create[F[_]: ConcurrentEffect: ContextShift: Timer](
cfg: Config,
termSignal: SignallingRef[F, Boolean],
connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, JoexApp[F]] =
for {
store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store).
withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])).
resource
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
.withTask(
JobTask.json(
ProcessItemArgs.taskName,
ItemHandler[F](cfg.extraction),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR

View File

@ -15,11 +15,17 @@ import scala.concurrent.ExecutionContext
object JoexServer {
private case class App[F[_]](
httpApp: HttpApp[F],
termSig: SignallingRef[F, Boolean],
exitRef: Ref[F, ExitCode]
)
private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode])
def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker)
(implicit T: Timer[F]): Stream[F, Nothing] = {
def stream[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F]): Stream[F, Nothing] = {
val app = for {
signal <- Resource.liftF(SignallingRef[F, Boolean](false))
@ -36,13 +42,14 @@ object JoexServer {
} yield App(finalHttpApp, signal, exitCode)
Stream.resource(app).flatMap(app =>
BlazeServerBuilder[F].
bindHttp(cfg.bind.port, cfg.bind.address).
withHttpApp(app.httpApp).
withoutBanner.
serveWhile(app.termSig, app.exitRef)
Stream
.resource(app)
.flatMap(app =>
BlazeServerBuilder[F]
.bindHttp(cfg.bind.port, cfg.bind.address)
.withHttpApp(app.httpApp)
.withoutBanner
.serveWhile(app.termSig, app.exitRef)
)
}.drain

View File

@ -14,10 +14,12 @@ object Main extends IOApp {
private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking")))
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect")))
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))
)
def run(args: List[String]) = {
args match {
@ -40,12 +42,15 @@ object Main extends IOApp {
}
val cfg = ConfigFile.loadConfig
val banner = Banner("JOEX"
, BuildInfo.version
, BuildInfo.gitHeadCommit
, cfg.jdbc.url
, Option(System.getProperty("config.file"))
, cfg.appId, cfg.baseUrl)
val banner = Banner(
"JOEX",
BuildInfo.version,
BuildInfo.gitHeadCommit,
cfg.jdbc.url,
Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}")
JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success)
}

View File

@ -23,20 +23,28 @@ object CreateItem {
Task { ctx =>
val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet
def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files).
flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))).
collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }).
zipWithIndex.
evalMap({ case (f, index) =>
Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
}).
compile.toVector
def fileMetas(itemId: Ident, now: Timestamp) =
Stream
.emits(ctx.args.files)
.flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
.collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f })
.zipWithIndex
.evalMap({
case (f, index) =>
Ident
.randomId[F]
.map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
})
.compile
.toVector
val item = RItem.newItem[F](ctx.args.meta.collective
, ctx.args.makeSubject
, ctx.args.meta.sourceAbbrev
, ctx.args.meta.direction.getOrElse(Direction.Incoming)
, ItemState.Premature)
val item = RItem.newItem[F](
ctx.args.meta.collective,
ctx.args.makeSubject,
ctx.args.meta.sourceAbbrev,
ctx.args.meta.direction.getOrElse(Direction.Incoming),
ItemState.Premature
)
for {
_ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)")
@ -56,16 +64,28 @@ object CreateItem {
Task { ctx =>
for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F]
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F]
rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid)))
} yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
else ().pure[F]
rms <- cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
} yield cand.headOption.map(ri =>
ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)
)
}
private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] =
private def logDifferences[F[_]: Sync](
ctx: Context[F, ProcessItemArgs],
saved: Vector[RAttachment],
saveCount: Int
): F[Unit] =
if (ctx.args.files.size != saved.size) {
ctx.logger.warn(s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount")
ctx.logger.warn(
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
)
} else {
().pure[F]
}

View File

@ -19,45 +19,65 @@ object FindProposal {
def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
val rmas = data.metas.map(rm =>
rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
ctx.logger.info("Starting find-proposal") *>
rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))).
flatMap(rmv => rmv.traverse(rm =>
rmas
.traverse(rm =>
processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))
)
.flatMap(rmv =>
rmv
.traverse(rm =>
ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *>
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))).
map(_ => data.copy(metas = rmv)))
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))
)
.map(_ => data.copy(metas = rmv))
)
}
def processAttachment[F[_]: Sync]( rm: RAttachmentMeta
, rd: Vector[NerDateLabel]
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
def processAttachment[F[_]: Sync](
rm: RAttachmentMeta,
rd: Vector[NerDateLabel],
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx))
List(finder.find(rm.nerlabels), makeDateProposal(rd)).
traverse(identity).map(MetaProposalList.flatten)
List(finder.find(rm.nerlabels), makeDateProposal(rd))
.traverse(identity)
.map(MetaProposalList.flatten)
}
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = {
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] =
Timestamp.current[F].map { now =>
val latestFirst = dates.sortWith(_.date isAfter _.date)
val latestFirst = dates.sortWith((l1, l2) => l1.date.isAfter(l2.date))
val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate
val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate))
val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate,
after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))))
val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate,
before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))))
val dueDates = MetaProposalList.fromSeq1(
MetaProposalType.DueDate,
after.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
val itemDates = MetaProposalList.fromSeq1(
MetaProposalType.DocDate,
before.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
MetaProposalList.flatten(Seq(dueDates, itemDates))
}
}
def removeDuplicates(labels: List[NerLabel]): List[NerLabel] =
labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) =>
labels
.foldLeft((Set.empty[String], List.empty[NerLabel])) {
case ((seen, result), el) =>
if (seen.contains(el.tag.name + el.label.toLowerCase)) (seen, result)
else (seen + (el.tag.name + el.label.toLowerCase), el :: result)
}._2.sortBy(_.startPosition)
}
._2
.sortBy(_.startPosition)
trait Finder[F[_]] { self =>
def find(labels: Seq[NerLabel]): F[MetaProposalList]
@ -80,12 +100,14 @@ object FindProposal {
else f.map(ml1 => ml0.fillEmptyFrom(ml1))
})
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)
(implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] =
flatMap(res0 => {
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
implicit F: FlatMap[F],
F2: Applicative[F]
): Finder[F] =
flatMap { res0 =>
if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0)
else f.map(res1 => res0.fillEmptyFrom(res1))
})
}
}
object Finder {
@ -102,7 +124,11 @@ object FindProposal {
labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten)
}
private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
private def search[F[_]: Sync](
nt: NerLabel,
exact: Boolean,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val value =
if (exact) normalizeSearchValue(nt.label)
else s"%${normalizeSearchValue(nt.label)}%"
@ -110,42 +136,51 @@ object FindProposal {
if (exact) 2 else 5
if (value.length < minLength) {
ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty)
} else nt.tag match {
ctx.logger
.debug(s"Skipping too small value '$value' (original '${nt.label}').")
.map(_ => MetaProposalList.empty)
} else
nt.tag match {
case NerTag.Organization =>
ctx.logger.debug(s"Looking for organizations: $value") *>
ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)).
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
ctx.store
.transact(ROrganization.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
case NerTag.Person =>
val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)).
map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val s1 = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, value, true))
.map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
val s2 = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking for persons: $value") *> (for {
ml0 <- s1
ml1 <- s2
} yield ml0 |+| ml1)
case NerTag.Location =>
ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.").
map(_ => MetaProposalList.empty)
ctx.logger
.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.")
.map(_ => MetaProposalList.empty)
case NerTag.Misc =>
ctx.logger.debug(s"Looking for equipments: $value") *>
ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)).
map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
ctx.store
.transact(REquipment.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
case NerTag.Email =>
searchContact(nt, ContactKind.Email, value, ctx)
case NerTag.Website =>
if (!exact) {
val searchString = Domain.domainFromUri(nt.label.toLowerCase).
toOption.
map(_.toPrimaryDomain.asString).
map(s => s"%$s%").
getOrElse(value)
val searchString = Domain
.domainFromUri(nt.label.toLowerCase)
.toOption
.map(_.toPrimaryDomain.asString)
.map(s => s"%$s%")
.getOrElse(value)
searchContact(nt, ContactKind.Website, searchString, ctx)
} else {
searchContact(nt, ContactKind.Website, value, ctx)
@ -157,16 +192,21 @@ object FindProposal {
}
}
private def searchContact[F[_]: Sync]( nt: NerLabel
, kind: ContactKind
, value: String
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = {
val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)).
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)).
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
private def searchContact[F[_]: Sync](
nt: NerLabel,
kind: ContactKind,
value: String,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val orgs = ctx.store
.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
val corrP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking with $kind: $value") *>
List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten)

View File

@ -4,10 +4,12 @@ import docspell.common.{Ident, NerDateLabel, NerLabel}
import docspell.joex.process.ItemData.AttachmentDates
import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
case class ItemData( item: RItem
, attachments: Vector[RAttachment]
, metas: Vector[RAttachmentMeta]
, dateLabels: Vector[AttachmentDates]) {
case class ItemData(
item: RItem,
attachments: Vector[RAttachment],
metas: Vector[RAttachmentMeta],
dateLabels: Vector[AttachmentDates]
) {
def findMeta(attachId: Ident): Option[RAttachmentMeta] =
metas.find(_.id == attachId)
@ -16,7 +18,6 @@ case class ItemData( item: RItem
dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty)
}
object ItemData {
case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) {

View File

@ -10,14 +10,13 @@ import docspell.text.ocr.{Config => OcrConfig}
object ItemHandler {
def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
logWarn("Now cancelling. Deleting potentially created data.").
flatMap(_ => deleteByFileIds)
logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ => deleteByFileIds)
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] =
CreateItem[F].
flatMap(itemStateTask(ItemState.Processing)).
flatMap(safeProcess[F](cfg)).
map(_ => ())
CreateItem[F]
.flatMap(itemStateTask(ItemState.Processing))
.flatMap(safeProcess[F](cfg))
.map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx =>
@ -30,22 +29,21 @@ object ItemHandler {
last = ctx.config.retries == current.getOrElse(0)
} yield last
def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
def safeProcess[F[_]: Sync: ContextShift](
cfg: OcrConfig
)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
case true =>
ProcessItem[F](cfg)(data).
attempt.flatMap({
ProcessItem[F](cfg)(data).attempt.flatMap({
case Right(d) =>
Task.pure(d)
case Left(ex) =>
logWarn[F]("Processing failed on last retry. Creating item but without proposals.").
flatMap(_ => itemStateTask(ItemState.Created)(data)).
andThen(_ => Sync[F].raiseError(ex))
logWarn[F]("Processing failed on last retry. Creating item but without proposals.")
.flatMap(_ => itemStateTask(ItemState.Created)(data))
.andThen(_ => Sync[F].raiseError(ex))
})
case false =>
ProcessItem[F](cfg)(data).
flatMap(itemStateTask(ItemState.Created))
ProcessItem[F](cfg)(data).flatMap(itemStateTask(ItemState.Created))
}
def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =

View File

@ -13,28 +13,40 @@ object LinkProposal {
val proposals = MetaProposalList.flatten(data.metas.map(_.proposals))
ctx.logger.info(s"Starting linking proposals") *>
MetaProposalType.all.
traverse(applyValue(data, proposals, ctx)).
map(result => ctx.logger.info(s"Results from proposal processing: $result")).
map(_ => data)
MetaProposalType.all
.traverse(applyValue(data, proposals, ctx))
.map(result => ctx.logger.info(s"Results from proposal processing: $result"))
.map(_ => data)
}
def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = {
def applyValue[F[_]: Sync](
data: ItemData,
proposalList: MetaProposalList,
ctx: Context[F, ProcessItemArgs]
)(mpt: MetaProposalType): F[Result] =
proposalList.find(mpt) match {
case None =>
Result.noneFound(mpt).pure[F]
case Some(a) if a.isSingleValue =>
ctx.logger.info(s"Found one candidate for ${a.proposalType}") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).
map(_ => Result.single(mpt))
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.single(mpt)
)
case Some(a) =>
ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).
map(_ => Result.multiple(mpt))
}
ctx.logger.info(
s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first."
) *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.multiple(mpt)
)
}
def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] =
def setItemMeta[F[_]: Sync](
itemId: Ident,
ctx: Context[F, ProcessItemArgs],
mpt: MetaProposalType,
value: Ident
): F[Int] =
mpt match {
case MetaProposalType.CorrOrg =>
ctx.logger.debug(s"Updating item organization with: ${value.id}") *>
@ -54,7 +66,6 @@ object LinkProposal {
ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0)
}
sealed trait Result {
def proposalType: MetaProposalType
}

View File

@ -7,13 +7,15 @@ import docspell.text.ocr.{Config => OcrConfig}
object ProcessItem {
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
TextExtraction(cfg, item).
flatMap(Task.setProgress(25)).
flatMap(TextAnalysis[F]).
flatMap(Task.setProgress(50)).
flatMap(FindProposal[F]).
flatMap(Task.setProgress(75)).
flatMap(LinkProposal[F]).
flatMap(Task.setProgress(99))
def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
TextExtraction(cfg, item)
.flatMap(Task.setProgress(25))
.flatMap(TextAnalysis[F])
.flatMap(Task.setProgress(50))
.flatMap(FindProposal[F])
.flatMap(Task.setProgress(75))
.flatMap(LinkProposal[F])
.flatMap(Task.setProgress(99))
}

View File

@ -17,9 +17,9 @@ object TestTasks {
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx =>
ctx.logger.info(s"Failing the task run :(").map(_ =>
sys.error("Oh, cannot extract gold from this document")
)
ctx.logger
.info(s"Failing the task run :(")
.map(_ => sys.error("Oh, cannot extract gold from this document"))
}
def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =

View File

@ -19,21 +19,26 @@ object TextAnalysis {
s <- Duration.stopTime[F]
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)))
_ <- t.traverse(m =>
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
)
e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector
} yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2))
}
def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
def annotateAttachment[F[_]: Sync](
lang: Language
)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
for {
list0 <- stanfordNer[F](lang, rm)
list1 <- contactNer[F](rm)
dates <- dateNer[F](rm, lang)
} yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates)
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay {
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] =
Sync[F].delay {
rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty)
}
@ -42,8 +47,10 @@ object TextAnalysis {
}
def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay {
AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty))
AttachmentDates(
rm,
rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)
)
}
}

View File

@ -11,7 +11,10 @@ import docspell.text.ocr.{TextExtract, Config => OcrConfig}
object TextExtraction {
def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] =
def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig,
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
for {
_ <- ctx.logger.info("Starting text extraction")
@ -24,22 +27,33 @@ object TextExtraction {
} yield item.copy(metas = txt)
}
def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] =
def extractTextToMeta[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: OcrConfig,
lang: Language
)(ra: RAttachment): F[RAttachmentMeta] =
for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}")
dst <- Duration.stopTime[F]
txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra)
meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty))
est <- dst
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}")
_ <- ctx.logger.debug(
s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}"
)
} yield meta
def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace.get(ra.fileId.id).
unNoneTerminate.
through(store.bitpeace.fetchData2(RangeDef.all))
def extractText[F[_]: Sync: ContextShift](
ocrConfig: OcrConfig,
lang: Language,
store: Store[F],
blocker: Blocker
)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.through(store.bitpeace.fetchData2(RangeDef.all))
TextExtract.extract(data, blocker, lang.iso3, ocrConfig).
compile.last
TextExtract.extract(data, blocker, lang.iso3, ocrConfig).compile.last
}
}

View File

@ -14,11 +14,15 @@ object InfoRoutes {
import dsl._
HttpRoutes.of[F] {
case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version
, BuildInfo.builtAtMillis
, BuildInfo.builtAtString
, BuildInfo.gitHeadCommit.getOrElse("")
, BuildInfo.gitDescribedVersion.getOrElse("")))
Ok(
VersionInfo(
BuildInfo.version,
BuildInfo.builtAtMillis,
BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
}
}
}

View File

@ -31,7 +31,9 @@ object JoexRoutes {
case POST -> Root / "shutdownAndExit" =>
for {
_ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown)
_ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp
@ -52,7 +54,15 @@ object JoexRoutes {
}
def mkJob(j: RJob): Job =
Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch))
Job(
j.id,
j.subject,
j.submitted,
j.priority,
j.retries,
j.progress,
j.started.getOrElse(Timestamp.Epoch)
)
def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog =
JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList)

View File

@ -32,20 +32,24 @@ trait Context[F[_], A] { self =>
object Context {
private[this] val log = getLogger
def create[F[_]: Functor, A]( job: RJob
, arg: A
, config: SchedulerConfig
, log: Logger[F]
, store: Store[F]
, blocker: Blocker): Context[F, A] =
def create[F[_]: Functor, A](
job: RJob,
arg: A,
config: SchedulerConfig,
log: Logger[F],
store: Store[F],
blocker: Blocker
): Context[F, A] =
new ContextImpl(arg, log, store, blocker, config, job.id)
def apply[F[_]: Concurrent, A]( job: RJob
, arg: A
, config: SchedulerConfig
, logSink: LogSink[F]
, blocker: Blocker
, store: Store[F]): F[Context[F, A]] =
def apply[F[_]: Concurrent, A](
job: RJob,
arg: A,
config: SchedulerConfig,
logSink: LogSink[F],
blocker: Blocker,
store: Store[F]
): F[Context[F, A]] =
for {
_ <- log.ftrace("Creating logger for task run")
logger <- Logger(job.id, job.info, config.logBufferSize, logSink)
@ -53,13 +57,14 @@ object Context {
ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx
private final class ContextImpl[F[_]: Functor, A]( val args: A
, val logger: Logger[F]
, val store: Store[F]
, val blocker: Blocker
, val config: SchedulerConfig
, val jobId: Ident)
extends Context[F,A] {
final private class ContextImpl[F[_]: Functor, A](
val args: A,
val logger: Logger[F],
val store: Store[F],
val blocker: Blocker,
val config: SchedulerConfig,
val jobId: Ident
) extends Context[F, A] {
def setProgress(percent: Int): F[Unit] = {
val pval = math.min(100, math.max(0, percent))

View File

@ -11,14 +11,13 @@ import docspell.common.Priority
*/
case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
def nextPriority: (CountingScheme, Priority) = {
def nextPriority: (CountingScheme, Priority) =
if (counter <= 0) (increment, Priority.High)
else {
val rest = counter % (high + low)
if (rest < high) (increment, Priority.High)
else (increment, Priority.Low)
}
}
def increment: CountingScheme =
copy(counter = counter + 1)
@ -32,8 +31,7 @@ object CountingScheme {
def readString(str: String): Either[String, CountingScheme] =
str.split(',') match {
case Array(h, l) =>
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).
left.map(_.getMessage)
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage)
case _ =>
Left(s"Invalid counting scheme: $str")
}

View File

@ -20,12 +20,15 @@ case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Tas
object JobTask {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])
(implicit D: Decoder[A]): JobTask[F] = {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])(
implicit D: Decoder[A]
): JobTask[F] = {
val convert: String => F[A] =
str => str.parseJsonAs[A] match {
str =>
str.parseJsonAs[A] match {
case Right(a) => a.pure[F]
case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
case Left(ex) =>
Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
}
JobTask(name, task.contramap(convert), onCancel.contramap(convert))

View File

@ -4,12 +4,14 @@ import cats.implicits._
import docspell.common._
import cats.effect.Sync
case class LogEvent( jobId: Ident
, jobInfo: String
, time: Timestamp
, level: LogLevel
, msg: String
, ex: Option[Throwable] = None) {
case class LogEvent(
jobId: Ident,
jobInfo: String,
time: Timestamp,
level: LogLevel,
msg: String,
ex: Option[Throwable] = None
) {
def logLine: String =
s">>> ${time.asString} $level $jobInfo: $msg"
@ -21,5 +23,4 @@ object LogEvent {
def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] =
Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg))
}

View File

@ -44,12 +44,22 @@ object LogSink {
LogSink(_.evalMap(e => logInternal(e)))
def db[F[_]: Sync](store: Store[F]): LogSink[F] =
LogSink(_.evalMap(ev => for {
LogSink(
_.evalMap(ev =>
for {
id <- Ident.randomId[F]
joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse(""))
joblog = RJobLog(
id,
ev.jobId,
ev.level,
ev.time,
ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("")
)
_ <- logInternal(ev)
_ <- store.transact(RJobLog.insert(joblog))
} yield ()))
} yield ()
)
)
def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = {
val s: Stream[F, Pipe[F, LogEvent, Unit]] =

View File

@ -33,13 +33,21 @@ object Logger {
LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1)
def error(ex: Throwable)(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1)
LogEvent
.create[F](jobId, jobInfo, LogLevel.Error, msg)
.map(le => le.copy(ex = Some(ex)))
.flatMap(q.enqueue1)
def error(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1)
}
def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] =
def apply[F[_]: Concurrent](
jobId: Ident,
jobInfo: String,
bufferSize: Int,
sink: LogSink[F]
): F[Logger[F]] =
for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q)

View File

@ -8,12 +8,13 @@ import docspell.store.queue.JobQueue
import fs2.concurrent.SignallingRef
case class SchedulerBuilder[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig
, tasks: JobTaskRegistry[F]
, store: Store[F]
, blocker: Blocker
, queue: Resource[F, JobQueue[F]]
, logSink: LogSink[F]) {
config: SchedulerConfig,
tasks: JobTaskRegistry[F],
store: Store[F],
blocker: Blocker,
queue: Resource[F, JobQueue[F]],
logSink: LogSink[F]
) {
def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] =
copy(config = cfg)
@ -33,7 +34,6 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] =
copy(logSink = sink)
def serve: Resource[F, Scheduler[F]] =
resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch))
@ -45,22 +45,25 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
perms <- Resource.liftF(Semaphore(config.poolSize.toLong))
} yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms)
scheduler.evalTap(_.init).
map(s => s: Scheduler[F])
scheduler.evalTap(_.init).map(s => s: Scheduler[F])
}
}
object SchedulerBuilder {
def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig
, blocker: Blocker
, store: Store[F]): SchedulerBuilder[F] =
new SchedulerBuilder[F](config
, JobTaskRegistry.empty[F]
, store
, blocker
, JobQueue(store)
, LogSink.db[F](store))
def apply[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig,
blocker: Blocker,
store: Store[F]
): SchedulerBuilder[F] =
new SchedulerBuilder[F](
config,
JobTaskRegistry.empty[F],
store,
blocker,
JobQueue(store),
LogSink.db[F](store)
)
}

View File

@ -2,24 +2,26 @@ package docspell.joex.scheduler
import docspell.common._
case class SchedulerConfig( name: Ident
, poolSize: Int
, countingScheme: CountingScheme
, retries: Int
, retryDelay: Duration
, logBufferSize: Int
, wakeupPeriod: Duration
case class SchedulerConfig(
name: Ident,
poolSize: Int,
countingScheme: CountingScheme,
retries: Int,
retryDelay: Duration,
logBufferSize: Int,
wakeupPeriod: Duration
)
object SchedulerConfig {
val default = SchedulerConfig(
name = Ident.unsafe("default-scheduler")
, poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
, countingScheme = CountingScheme(2, 1)
, retries = 5
, retryDelay = Duration.seconds(30)
, logBufferSize = 500
, wakeupPeriod = Duration.minutes(10)
name = Ident.unsafe("default-scheduler"),
poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
,
countingScheme = CountingScheme(2, 1),
retries = 5,
retryDelay = Duration.seconds(30),
logBufferSize = 500,
wakeupPeriod = Duration.minutes(10)
)
}

View File

@ -14,15 +14,17 @@ import SchedulerImpl._
import docspell.store.Store
import docspell.store.queries.QJob
final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig
, blocker: Blocker
, queue: JobQueue[F]
, tasks: JobTaskRegistry[F]
, store: Store[F]
, logSink: LogSink[F]
, state: SignallingRef[F, State[F]]
, waiter: SignallingRef[F, Boolean]
, permits: Semaphore[F]) extends Scheduler[F] {
final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
val config: SchedulerConfig,
blocker: Blocker,
queue: JobQueue[F],
tasks: JobTaskRegistry[F],
store: Store[F],
logSink: LogSink[F],
state: SignallingRef[F, State[F]],
waiter: SignallingRef[F, Boolean],
permits: Semaphore[F]
) extends Scheduler[F] {
private[this] val logger = getLogger
@ -34,8 +36,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.runningToWaiting(config.name, store)
def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] =
ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala).
evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain)
ConcurrentEffect[F].start(
Stream
.awakeEvery[F](config.wakeupPeriod.toScala)
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
.compile
.drain
)
def getRunning: F[Vector[RJob]] =
state.get.flatMap(s => QJob.findAll(s.getRunning, store))
@ -51,21 +58,23 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def shutdown(cancelAll: Boolean): F[Unit] = {
val doCancel =
state.get.
flatMap(_.cancelTokens.values.toList.traverse(identity)).
map(_ => ())
state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ())
val runShutdown =
state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F])
val wait = Stream.eval(runShutdown).
evalMap(_ => logger.finfo("Scheduler is shutting down now.")).
flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))).
flatMap(state => {
val wait = Stream
.eval(runShutdown)
.evalMap(_ => logger.finfo("Scheduler is shutting down now."))
.flatMap(_ =>
Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
)
.flatMap { state =>
if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running."))
else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
else
Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
Stream.emit(state)
})
}
(wait.drain ++ Stream.emit(())).compile.lastOrError
}
@ -82,15 +91,24 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
_ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay)
else
queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined
Stream.eval(state.get.map(_.shutdownRequest)).
evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]).
flatMap(if (_) Stream.empty else Stream.eval(body)).
flatMap({
Stream
.eval(state.get.map(_.shutdownRequest))
.evalTap(
if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
else ().pure[F]
)
.flatMap(if (_) Stream.empty else Stream.eval(body))
.flatMap({
case true =>
mainLoop
case false =>
@ -103,7 +121,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def execute(job: RJob): F[Unit] = {
val task = for {
jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}")
jobtask <- tasks
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask
task match {
@ -123,7 +143,9 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onFinish(job: RJob, finalState: JobState): F[Unit] =
for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)"))
_ <- permits.release *> permits.available.flatMap(a =>
logger.fdebug(s"Permit released ($a free)")
)
_ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store)
} yield ()
@ -131,9 +153,14 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onStart(job: RJob): F[Unit] =
QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck
def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = {
task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)).
mapF(_.attempt.flatMap({
def wrapTask(
job: RJob,
task: Task[F, String, Unit],
ctx: Context[F, String]
): Task[F, String, Unit] =
task
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa))
.mapF(_.attempt.flatMap({
case Right(()) =>
logger.info(s"Job execution successful: ${job.info}")
ctx.logger.info("Job execution successful") *>
@ -148,16 +175,18 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.exceedsRetries(job.id, config.retries, store).flatMap {
case true =>
logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.").
map(_ => JobState.Failed: JobState)
ctx.logger
.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
.map(_ => JobState.Failed: JobState)
case false =>
logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.").
map(_ => JobState.Stuck: JobState)
ctx.logger
.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
.map(_ => JobState.Stuck: JobState)
}
}
})).
mapF(_.attempt.flatMap {
}))
.mapF(_.attempt.flatMap {
case Right(jstate) =>
onFinish(job, jstate)
case Left(ex) =>
@ -166,13 +195,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
// since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways
onFinish(job, JobState.Stuck)
})
}
def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = {
val bfa = blocker.blockOn(code)
logger.fdebug(s"Forking job ${job.info}") *>
ConcurrentEffect[F].start(bfa).
map(fiber =>
ConcurrentEffect[F]
.start(bfa)
.map(fiber =>
logger.fdebug(s"Cancelling job ${job.info}") *>
fiber.cancel *>
onCancel.attempt.map({
@ -184,7 +213,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
state.modify(_.markCancelled(job)) *>
onFinish(job, JobState.Cancelled) *>
ctx.logger.warn("Job has been cancelled.") *>
logger.fdebug(s"Job ${job.info} has been cancelled."))
logger.fdebug(s"Job ${job.info} has been cancelled.")
)
}
}
@ -193,10 +223,12 @@ object SchedulerImpl {
def emptyState[F[_]]: State[F] =
State(Map.empty, Set.empty, Map.empty, false)
case class State[F[_]]( counters: Map[Ident, CountingScheme]
, cancelled: Set[Ident]
, cancelTokens: Map[Ident, CancelToken[F]]
, shutdownRequest: Boolean) {
case class State[F[_]](
counters: Map[Ident, CountingScheme],
cancelled: Set[Ident],
cancelTokens: Map[Ident, CancelToken[F]],
shutdownRequest: Boolean
) {
def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = {
val (cs, prio) = counters.getOrElse(group, initial).nextPriority

View File

@ -27,8 +27,8 @@ trait Task[F[_], A, B] {
def attempt(implicit F: ApplicativeError[F, Throwable]): Task[F, A, Either[Throwable, B]] =
mapF(_.attempt)
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = {
ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { ctxc: Context[F, C] =>
f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
}
}
@ -46,7 +46,6 @@ object Task {
def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] =
c => k.run(c)
def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] =
Kleisli(t.run)

View File

@ -7,27 +7,37 @@ import docspell.backend.{Config => BackendConfig}
import docspell.common._
import scodec.bits.ByteVector
case class Config(appName: String
, appId: Ident
, baseUrl: LenientUri
, bind: Config.Bind
, backend: BackendConfig
, auth: Login.Config
case class Config(
appName: String,
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
backend: BackendConfig,
auth: Login.Config
)
object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "")
val postgres =
JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
val default: Config =
Config("Docspell"
, Ident.unsafe("restserver1")
, LenientUri.unsafe("http://localhost:7880")
, Config.Bind("localhost", 7880)
, BackendConfig(postgres
, SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24))
, BackendConfig.Files(512 * 1024, List(MimeType.pdf)))
, Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2)))
Config(
"Docspell",
Ident.unsafe("restserver1"),
LenientUri.unsafe("http://localhost:7880"),
Config.Bind("localhost", 7880),
BackendConfig(
postgres,
SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)),
BackendConfig.Files(512 * 1024, List(MimeType.pdf))
),
Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))
)
case class Bind(address: String, port: Int)
}

View File

@ -13,12 +13,14 @@ import org.log4s._
object Main extends IOApp {
private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool(
ThreadFactories.ofName("docspell-restserver-blocking")))
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-restserver-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5,
ThreadFactories.ofName("docspell-dbconnect")))
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-dbconnect"))
)
def run(args: List[String]) = {
args match {
@ -41,12 +43,15 @@ object Main extends IOApp {
}
val cfg = ConfigFile.loadConfig
val banner = Banner("REST Server"
, BuildInfo.version
, BuildInfo.gitHeadCommit
, cfg.backend.jdbc.url
, Option(System.getProperty("config.file"))
, cfg.appId, cfg.baseUrl)
val banner = Banner(
"REST Server",
BuildInfo.version,
BuildInfo.gitHeadCommit,
cfg.backend.jdbc.url,
Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}")
RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success)
}

View File

@ -7,7 +7,8 @@ import docspell.common.NodeType
import scala.concurrent.ExecutionContext
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] {
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F])
extends RestApp[F] {
def init: F[Unit] =
backend.node.register(config.appId, NodeType.Restserver, config.baseUrl)
@ -18,7 +19,12 @@ final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[
object RestAppImpl {
def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] =
def create[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, RestApp[F]] =
for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend)

View File

@ -15,8 +15,12 @@ import scala.concurrent.ExecutionContext
object RestServer {
def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker)
(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
def stream[F[_]: ConcurrentEffect](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
val app = for {
restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker)
@ -24,8 +28,8 @@ object RestServer {
httpApp = Router(
"/api/info" -> routes.InfoRoutes(),
"/api/v1/open/" -> openRoutes(cfg, restApp),
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) {
token => securedRoutes(cfg, restApp, token)
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
securedRoutes(cfg, restApp, token)
},
"/app/assets" -> WebjarRoutes.appRoutes[F](blocker),
"/app" -> TemplateRoutes[F](blocker, cfg)
@ -35,16 +39,22 @@ object RestServer {
} yield finalHttpApp
Stream.resource(app).flatMap(httpApp =>
BlazeServerBuilder[F].
bindHttp(cfg.bind.port, cfg.bind.address).
withHttpApp(httpApp).
withoutBanner.
serve)
Stream
.resource(app)
.flatMap(httpApp =>
BlazeServerBuilder[F]
.bindHttp(cfg.bind.port, cfg.bind.address)
.withHttpApp(httpApp)
.withoutBanner
.serve
)
}.drain
def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] =
def securedRoutes[F[_]: Effect](
cfg: Config,
restApp: RestApp[F],
token: AuthToken
): HttpRoutes[F] =
Router(
"auth" -> LoginRoutes.session(restApp.backend.login, cfg),
"tag" -> TagRoutes(restApp.backend, token),

View File

@ -14,7 +14,14 @@ case class CookieData(auth: AuthToken) {
val domain = cfg.baseUrl.host
val sec = cfg.baseUrl.scheme.exists(_.endsWith("s"))
val path = cfg.baseUrl.path / "api" / "v1" / "sec"
ResponseCookie(CookieData.cookieName, asString, domain = domain, path = Some(path.asString), httpOnly = true, secure = sec)
ResponseCookie(
CookieData.cookieName,
asString,
domain = domain,
path = Some(path.asString),
httpOnly = true,
secure = sec
)
}
}
object CookieData {
@ -22,18 +29,21 @@ object CookieData {
val headerName = "X-Docspell-Auth"
def authenticator[F[_]](r: Request[F]): Either[String, String] =
fromCookie(r) orElse fromHeader(r)
fromCookie(r).orElse(fromHeader(r))
def fromCookie[F[_]](req: Request[F]): Either[String, String] = {
def fromCookie[F[_]](req: Request[F]): Either[String, String] =
for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie")
cookie <- header.values.toList
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
} yield cookie.content
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] = {
req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator")
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] =
req.headers
.get(CaseInsensitiveString(headerName))
.map(_.value)
.toRight("Couldn't find an authenticator")
def deleteCookie(cfg: Config): ResponseCookie =
ResponseCookie(

View File

@ -24,31 +24,37 @@ trait Conversions {
// insights
def mkItemInsights(d: InsightData): ItemInsights =
ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2))))
ItemInsights(
d.incoming,
d.outgoing,
d.bytes,
TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))
)
// attachment meta
def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta =
AttachmentMeta(rm.content.getOrElse("")
, rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition))
, mkItemProposals(rm.proposals))
AttachmentMeta(
rm.content.getOrElse(""),
rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)),
mkItemProposals(rm.proposals)
)
// item proposal
def mkItemProposals(ml: MetaProposalList): ItemProposals = {
def get(mpt: MetaProposalType) =
ml.find(mpt).
map(mp => mp.values.toList.map(_.ref).map(mkIdName)).
getOrElse(Nil)
ml.find(mpt).map(mp => mp.values.toList.map(_.ref).map(mkIdName)).getOrElse(Nil)
def getDates(mpt: MetaProposalType): List[Timestamp] =
ml.find(mpt).
map(mp => mp.values.toList.
map(cand => cand.ref.id.id).
flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption).
map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))).
map(zdt => Timestamp(zdt.toInstant))).
getOrElse(Nil).
distinct.
take(5)
ml.find(mpt)
.map(mp =>
mp.values.toList
.map(cand => cand.ref.id.id)
.flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption)
.map(_.atTime(12, 0).atZone(ZoneId.of("GMT")))
.map(zdt => Timestamp(zdt.toInstant))
)
.getOrElse(Nil)
.distinct
.take(5)
ItemProposals(
corrOrg = get(MetaProposalType.CorrOrg),
@ -62,23 +68,25 @@ trait Conversions {
// item detail
def mkItemDetail(data: OItem.ItemData): ItemDetail =
ItemDetail(data.item.id
, data.item.direction
, data.item.name
, data.item.source
, data.item.state
, data.item.created
, data.item.updated
, data.item.itemDate
, data.corrOrg.map(o => IdName(o.oid, o.name))
, data.corrPerson.map(p => IdName(p.pid, p.name))
, data.concPerson.map(p => IdName(p.pid, p.name))
, data.concEquip.map(e => IdName(e.eid, e.name))
, data.inReplyTo.map(mkIdName)
, data.item.dueDate
, data.item.notes
, data.attachments.map((mkAttachment _).tupled).toList
, data.tags.map(mkTag).toList)
ItemDetail(
data.item.id,
data.item.direction,
data.item.name,
data.item.source,
data.item.state,
data.item.created,
data.item.updated,
data.item.itemDate,
data.corrOrg.map(o => IdName(o.oid, o.name)),
data.corrPerson.map(p => IdName(p.pid, p.name)),
data.concPerson.map(p => IdName(p.pid, p.name)),
data.concEquip.map(e => IdName(e.eid, e.name)),
data.inReplyTo.map(mkIdName),
data.item.dueDate,
data.item.notes,
data.attachments.map((mkAttachment _).tupled).toList,
data.tags.map(mkTag).toList
)
def mkAttachment(ra: RAttachment, m: FileMeta): Attachment =
Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString))
@ -86,20 +94,21 @@ trait Conversions {
// item list
def mkQuery(m: ItemSearch, coll: Ident): OItem.Query =
OItem.Query(coll
, m.name
, if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed)
, m.direction
, m.corrPerson
, m.corrOrg
, m.concPerson
, m.concEquip
, m.tagsInclude.map(Ident.unsafe)
, m.tagsExclude.map(Ident.unsafe)
, m.dateFrom
, m.dateUntil
, m.dueDateFrom
, m.dueDateUntil
OItem.Query(
coll,
m.name,
if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed),
m.direction,
m.corrPerson,
m.corrOrg,
m.concPerson,
m.concEquip,
m.tagsInclude.map(Ident.unsafe),
m.tagsExclude.map(Ident.unsafe),
m.dateFrom,
m.dateUntil,
m.dueDateFrom,
m.dueDateUntil
)
def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = {
@ -113,8 +122,20 @@ trait Conversions {
}
def mkItemLight(i: OItem.ListItem): ItemLight =
ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount)
ItemLight(
i.id,
i.name,
i.state,
i.date,
i.dueDate,
i.source,
i.direction.name.some,
i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName),
i.concPerson.map(mkIdName),
i.concEquip.map(mkIdName),
i.fileCount
)
// job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
@ -128,46 +149,57 @@ trait Conversions {
val t2 = f(j2).getOrElse(Timestamp.Epoch)
t1.value.isBefore(t2.value)
}
JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started))
, state.done.map(mkJobDetail).toList.sortWith(desc(_.finished))
, state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some)))
JobQueueState(
state.running.map(mkJobDetail).toList.sortWith(asc(_.started)),
state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)),
state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))
)
}
def mkJobDetail(jd: OJob.JobDetail): JobDetail =
JobDetail(jd.job.id
, jd.job.subject
, jd.job.submitted
, jd.job.priority
, jd.job.state
, jd.job.retries
, jd.logs.map(mkJobLog).toList
, jd.job.progress
, jd.job.worker
, jd.job.started
, jd.job.finished)
JobDetail(
jd.job.id,
jd.job.subject,
jd.job.submitted,
jd.job.priority,
jd.job.state,
jd.job.retries,
jd.logs.map(mkJobLog).toList,
jd.job.progress,
jd.job.worker,
jd.job.started,
jd.job.finished
)
def mkJobLog(jl: RJobLog): JobLogEvent =
JobLogEvent(jl.created, jl.level, jl.message)
// upload
def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = {
body.through(fs2.text.utf8Decode).
parseJsonAs[ItemUploadMeta].
map(_.fold(ex => {
def readMultipart[F[_]: Effect](
mp: Multipart[F],
logger: Logger,
prio: Priority,
validFileTypes: Seq[MimeType]
): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] =
body
.through(fs2.text.utf8Decode)
.parseJsonAs[ItemUploadMeta]
.map(_.fold(ex => {
logger.error(ex)("Reading upload metadata failed.")
throw ex
}, identity))
}
val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")).
map(p => parseMeta(p.body)).
map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))).
getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val meta: F[(Boolean, UploadMeta)] = mp.parts
.find(_.name.exists(_.equalsIgnoreCase("meta")))
.map(p => parseMeta(p.body))
.map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes))))
.getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val files = mp.parts.
filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))).
map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body))
val files = mp.parts
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
.map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
)
for {
metaData <- meta
_ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData"))
@ -178,8 +210,14 @@ trait Conversions {
// organization and person
def mkOrg(v: OOrganization.OrgAndContacts): Organization = {
val ro = v.org
Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList, ro.notes, ro.created)
Organization(
ro.oid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.created
)
}
def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = {
@ -189,7 +227,17 @@ trait Conversions {
now <- Timestamp.current[F]
oid <- Ident.randomId[F]
cont <- contacts(oid)
org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now)
org = ROrganization(
oid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
now
)
} yield OOrganization.OrgAndContacts(org, cont)
}
@ -198,14 +246,31 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, oid.some, None))
for {
cont <- contacts(v.id)
org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created)
org = ROrganization(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.created
)
} yield OOrganization.OrgAndContacts(org, cont)
}
def mkPerson(v: OOrganization.PersonAndContacts): Person = {
val ro = v.person
Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created)
Person(
ro.pid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.concerning,
ro.created
)
}
def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = {
@ -215,7 +280,18 @@ trait Conversions {
now <- Timestamp.current[F]
pid <- Ident.randomId[F]
cont <- contacts(pid)
org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now)
org = RPerson(
pid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
now
)
} yield OOrganization.PersonAndContacts(org, cont)
}
@ -224,7 +300,18 @@ trait Conversions {
v.contacts.traverse(c => newContact(c, None, pid.some))
for {
cont <- contacts(v.id)
org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created)
org = RPerson(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
v.created
)
} yield OOrganization.PersonAndContacts(org, cont)
}
@ -233,7 +320,8 @@ trait Conversions {
Contact(rc.contactId, rc.value, rc.kind)
def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] =
timeId.map { case (id, now) =>
timeId.map {
case (id, now) =>
RContact(id, c.value, c.kind, pid, oid, now)
}
@ -242,12 +330,33 @@ trait Conversions {
User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created)
def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] =
timeId.map { case (id, now) =>
RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, 0, None, now)
timeId.map {
case (id, now) =>
RUser(
id,
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
0,
None,
now
)
}
def changeUser(u: User, cid: Ident): RUser =
RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created)
RUser(
Ident.unsafe(""),
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
u.loginCount,
u.lastLogin,
u.created
)
// tags
@ -255,21 +364,22 @@ trait Conversions {
Tag(rt.tagId, rt.name, rt.category, rt.created)
def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] =
timeId.map { case (id, now) =>
timeId.map {
case (id, now) =>
RTag(id, cid, t.name, t.category, now)
}
def changeTag(t: Tag, cid: Ident): RTag =
RTag(t.id, cid, t.name, t.category, t.created)
// sources
def mkSource(s: RSource): Source =
Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map({ case (id, now) =>
timeId.map({
case (id, now) =>
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
})
@ -281,7 +391,8 @@ trait Conversions {
Equipment(re.eid, re.name, re.created)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map({ case (id, now) =>
timeId.map({
case (id, now) =>
REquipment(id, cid, e.name, now)
})
@ -298,7 +409,8 @@ trait Conversions {
def basicResult(cr: JobCancelResult): BasicResult =
cr match {
case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.CancelRequested =>
BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.")
}
@ -317,7 +429,8 @@ trait Conversions {
def basicResult(cr: PassChangeResult): BasicResult = cr match {
case PassChangeResult.Success => BasicResult(true, "Password changed.")
case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.")
case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.")
case PassChangeResult.PasswordMismatch =>
BasicResult(false, "The current password is incorrect.")
case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
}

View File

@ -8,28 +8,26 @@ import org.http4s.dsl.Http4sDsl
trait ResponseGenerator[F[_]] {
self: Http4sDsl[F] =>
implicit final class EitherResponses[A, B](e: Either[A, B]) {
def toResponse(headers: Header*)
(implicit F: Applicative[F]
, w0: EntityEncoder[F, A]
, w1: EntityEncoder[F, B]): F[Response[F]] =
def toResponse(headers: Header*)(
implicit F: Applicative[F],
w0: EntityEncoder[F, A],
w1: EntityEncoder[F, B]
): F[Response[F]] =
e.fold(
a => UnprocessableEntity(a),
b => Ok(b)
).map(_.withHeaders(headers: _*))
)
.map(_.withHeaders(headers: _*))
}
implicit final class OptionResponse[A](o: Option[A]) {
def toResponse(headers: Header*)
(implicit F: Applicative[F]
, w0: EntityEncoder[F, A]): F[Response[F]] =
def toResponse(
headers: Header*
)(implicit F: Applicative[F], w0: EntityEncoder[F, A]): F[Response[F]] =
o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*))
}
}
object ResponseGenerator {
}
object ResponseGenerator {}

View File

@ -25,10 +25,11 @@ object AttachmentRoutes {
val mt = MediaType.unsafeParse(data.meta.mimetype.asString)
val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length)
val eTag: Header = ETag(data.meta.checksum)
val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
Ok(data.data.take(data.meta.length)).
map(r => r.withContentType(`Content-Type`(mt)).
withHeaders(cntLen, eTag, disp))
val disp: Header =
`Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
Ok(data.data.take(data.meta.length)).map(r =>
r.withContentType(`Content-Type`(mt)).withHeaders(cntLen, eTag, disp)
)
}
HttpRoutes.of {
@ -38,7 +39,8 @@ object AttachmentRoutes {
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData, inm)
resp <- if (matches) NotModified()
else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
else
fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
case GET -> Root / Ident(id) / "meta" =>
@ -50,8 +52,10 @@ object AttachmentRoutes {
}
}
private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]]
, noneMatch: Option[NonEmptyList[EntityTag]]): Boolean =
private def matchETag[F[_]](
fileData: Option[OItem.AttachmentData[F]],
noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean =
(fileData, noneMatch) match {
case (Some(fd), Some(nm)) =>
fd.meta.checksum == nm.head.tag

View File

@ -12,14 +12,17 @@ import org.http4s.server._
object Authenticate {
def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] =
def authenticateRequest[F[_]: Effect](
auth: String => F[Login.Result]
)(req: Request[F]): F[Login.Result] =
CookieData.authenticator(req) match {
case Right(str) => auth(str)
case Left(_) => Login.Result.invalidAuth.pure[F]
}
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = {
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(
pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._
@ -34,7 +37,9 @@ object Authenticate {
middleware(AuthedRoutes.of(pf))
}
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = {
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(
f: AuthToken => HttpRoutes[F]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._
@ -49,6 +54,8 @@ object Authenticate {
middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req)))
}
private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] =
private def getUser[F[_]: Effect](
auth: String => F[Login.Result]
): Kleisli[F, Request[F], Either[String, AuthToken]] =
Kleisli(r => authenticateRequest(auth)(r).map(_.toEither))
}

View File

@ -14,11 +14,15 @@ object InfoRoutes {
import dsl._
HttpRoutes.of[F] {
case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version
, BuildInfo.builtAtMillis
, BuildInfo.builtAtString
, BuildInfo.gitHeadCommit.getOrElse("")
, BuildInfo.gitDescribedVersion.getOrElse("")))
Ok(
VersionInfo(
BuildInfo.version,
BuildInfo.builtAtMillis,
BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
}
}
}

View File

@ -33,22 +33,36 @@ object LoginRoutes {
HttpRoutes.of[F] {
case req @ POST -> Root / "session" =>
Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req).
flatMap(res => makeResponse(dsl, cfg, res, ""))
Authenticate
.authenticateRequest(S.loginSession(cfg.auth))(req)
.flatMap(res => makeResponse(dsl, cfg, res, ""))
case POST -> Root / "logout" =>
Ok().map(_.addCookie(CookieData.deleteCookie(cfg)))
}
}
def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = {
def makeResponse[F[_]: Effect](
dsl: Http4sDsl[F],
cfg: Config,
res: Login.Result,
account: String
): F[Response[F]] = {
import dsl._
res match {
case Login.Result.Ok(token) =>
for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)).
map(_.addCookie(cd.asCookie(cfg)))
resp <- Ok(
AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
} yield resp
case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L))

View File

@ -6,9 +6,10 @@ import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher
object ParamDecoder {
implicit val booleanDecoder: QueryParamDecoder[Boolean] =
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean")
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_.equalsIgnoreCase("true")))(
"Boolean"
)
object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full")
}

View File

@ -47,7 +47,6 @@ object RegisterRoutes {
InviteResult(false, "Password is invalid.", None)
}
def convert(r: SignupResult): BasicResult = r match {
case SignupResult.CollectiveExists =>
BasicResult(false, "A collective with this name already exists.")
@ -62,7 +61,6 @@ object RegisterRoutes {
BasicResult(true, "Signup successful")
}
def convert(r: Registration): RegisterData =
RegisterData(r.collectiveName, r.login, r.password, r.invite)
}

View File

@ -26,7 +26,12 @@ object UploadRoutes {
case req @ POST -> Root / "item" =>
for {
multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes)
updata <- readMultipart(
multipart,
logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result))
} yield res

View File

@ -23,7 +23,11 @@ object UserRoutes {
case req @ POST -> Root / "changePassword" =>
for {
data <- req.as[PasswordChange]
res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword)
res <- backend.collective.changePassword(
user.account,
data.currentPassword,
data.newPassword
)
resp <- Ok(basicResult(res))
} yield resp

View File

@ -8,14 +8,21 @@ import docspell.backend.signup.{Config => SignupConfig}
import yamusca.imports._
import yamusca.implicits._
case class Flags( appName: String
, baseUrl: LenientUri
, signupMode: SignupConfig.Mode
, docspellAssetPath: String)
case class Flags(
appName: String,
baseUrl: LenientUri,
signupMode: SignupConfig.Mode,
docspellAssetPath: String
)
object Flags {
def apply(cfg: Config): Flags =
Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode, s"assets/docspell-webapp/${BuildInfo.version}")
Flags(
cfg.appName,
cfg.baseUrl,
cfg.backend.signup.mode,
s"assets/docspell-webapp/${BuildInfo.version}"
)
implicit val jsonEncoder: Encoder[Flags] =
deriveEncoder[Flags]

View File

@ -21,7 +21,9 @@ object TemplateRoutes {
val `text/html` = new MediaType("text", "html")
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit C: ContextShift[F]): HttpRoutes[F] = {
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(
implicit C: ContextShift[F]
): HttpRoutes[F] = {
val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker)))
val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker)))
@ -41,20 +43,21 @@ object TemplateRoutes {
}
}
def loadResource[F[_]: Sync](name: String): F[URL] = {
def loadResource[F[_]: Sync](name: String): F[URL] =
Option(getClass.getResource(name)) match {
case None =>
Sync[F].raiseError(new Exception("Unknown resource: " + name))
case Some(r) =>
r.pure[F]
}
}
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] =
Stream.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())).
flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)).
through(text.utf8Decode).
compile.fold("")(_ + _)
Stream
.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
.flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false))
.through(text.utf8Decode)
.compile
.fold("")(_ + _)
def parseTemplate[F[_]: Sync](str: String): F[Template] =
Sync[F].delay {
@ -64,47 +67,54 @@ object TemplateRoutes {
}
}
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[Template] = {
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).
map(t => {
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(
implicit C: ContextShift[F]
): F[Template] =
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
logger.info(s"Compiled template $url")
t
})
}
case class DocData(swaggerRoot: String, openapiSpec: String)
object DocData {
def apply(): DocData =
DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml")
DocData(
"/app/assets" + Webjars.swaggerui,
s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml"
)
implicit def yamuscaValueConverter: ValueConverter[DocData] =
ValueConverter.deriveConverter[DocData]
}
case class IndexData(flags: Flags
, cssUrls: Seq[String]
, jsUrls: Seq[String]
, faviconBase: String
, appExtraJs: String
, flagsJson: String)
case class IndexData(
flags: Flags,
cssUrls: Seq[String],
jsUrls: Seq[String],
faviconBase: String,
appExtraJs: String,
flagsJson: String
)
object IndexData {
def apply(cfg: Config): IndexData =
IndexData(Flags(cfg)
, Seq(
IndexData(
Flags(cfg),
Seq(
"/app/assets" + Webjars.semanticui + "/semantic.min.css",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css"
)
, Seq(
),
Seq(
"/app/assets" + Webjars.jquery + "/jquery.min.js",
"/app/assets" + Webjars.semanticui + "/semantic.min.js",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js"
),
s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js",
Flags(cfg).asJson.spaces2
)
, s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon"
, s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js"
, Flags(cfg).asJson.spaces2 )
implicit def yamuscaValueConverter: ValueConverter[IndexData] =
ValueConverter.deriveConverter[IndexData]
@ -116,10 +126,10 @@ object TemplateRoutes {
Option(ref.get) match {
case Some(a) => a.pure[F]
case None =>
fa.map(a => {
fa.map { a =>
ref.set(a)
a
})
}
}
}
}

View File

@ -9,7 +9,7 @@ import org.http4s.server.staticcontent.WebjarService.{WebjarAsset, Config => Web
object WebjarRoutes {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] =
webjarService(
WebjarConfig(
filter = assetFilter,
@ -17,10 +17,23 @@ object WebjarRoutes {
cacheStrategy = NoopCacheStrategy[F]
)
)
}
def assetFilter(asset: WebjarAsset): Boolean =
List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml").
exists(e => asset.asset.endsWith(e))
List(
".js",
".css",
".html",
".json",
".jpg",
".png",
".eot",
".woff",
".woff2",
".svg",
".otf",
".ttf",
".yml",
".xml"
).exists(e => asset.asset.endsWith(e))
}

View File

@ -2,10 +2,7 @@ package docspell.store
import docspell.common.LenientUri
case class JdbcConfig(url: LenientUri
, user: String
, password: String
) {
case class JdbcConfig(url: LenientUri, user: String, password: String) {
val dbmsName: Option[String] =
JdbcConfig.extractDbmsName(url)

View File

@ -22,16 +22,20 @@ trait Store[F[_]] {
object Store {
def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig
, connectEC: ExecutionContext
, blocker: Blocker): Resource[F, Store[F]] = {
def create[F[_]: Effect: ContextShift](
jdbc: JdbcConfig,
connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, Store[F]] = {
val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass
, jdbc.url.asString
, jdbc.user
, jdbc.password
, connectEC
, blocker)
val hxa = HikariTransactor.newHikariTransactor[F](
jdbc.driverClass,
jdbc.url.asString,
jdbc.user,
jdbc.password,
connectEC,
blocker
)
for {
xa <- hxa

View File

@ -21,7 +21,9 @@ trait DoobieMeta {
})
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces)
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
e.apply(a).noSpaces
)
implicit val metaCollectiveState: Meta[CollectiveState] =
Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString)
@ -45,7 +47,9 @@ trait DoobieMeta {
Meta[String].imap(JobState.unsafe)(_.name)
implicit val metaDirection: Meta[Direction] =
Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d))
Meta[Boolean].imap(flag =>
if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction
)(d => Direction.isIncoming(d))
implicit val metaPriority: Meta[Priority] =
Meta[Int].imap(Priority.fromInt)(Priority.toInt)

View File

@ -19,7 +19,9 @@ trait DoobieSyntax {
commas(fa :: fas.toList)
def and(fs: Seq[Fragment]): Fragment =
Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
Fragment.const(" (") ++ fs
.filter(f => !isEmpty(f))
.reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
def and(f0: Fragment, fs: Fragment*): Fragment =
and(f0 :: fs.toList)
@ -48,8 +50,9 @@ trait DoobieSyntax {
def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment =
Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")"))
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(
vals.map(f => sql"(" ++ f ++ sql")")
)
def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment =
selectSimple(commas(cols.map(_.f)), table, where)
@ -62,7 +65,6 @@ trait DoobieSyntax {
Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++
Fragment.const(") FROM ") ++ table ++ this.where(where)
// def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment =
// selectSimple(cols.map(_.prefix("a"))
// , table ++ fr"a," ++ RCollective.table ++ fr"b"
@ -70,11 +72,12 @@ trait DoobieSyntax {
// else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b")))
def selectCount(col: Column, table: Fragment, where: Fragment): Fragment =
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where)
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(
where
)
def deleteFrom(table: Fragment, where: Fragment): Fragment = {
def deleteFrom(table: Fragment, where: Fragment): Fragment =
fr"DELETE FROM" ++ table ++ this.where(where)
}
def withCTE(ps: (String, Fragment)*): Fragment = {
val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")")

View File

@ -1,5 +1,3 @@
package docspell.store.impl
object Implicits extends DoobieMeta
with DoobieSyntax
object Implicits extends DoobieMeta with DoobieSyntax

View File

@ -10,7 +10,8 @@ import doobie._
import doobie.implicits._
final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] {
val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
val bitpeaceCfg =
BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
def migrate: F[Int] =
FlywayMigrate.run[F](jdbc)
@ -24,14 +25,14 @@ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends
def bitpeace: Bitpeace[F] =
Bitpeace(bitpeaceCfg, xa)
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = {
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] =
for {
save <- transact(insert).attempt
exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b)))
} yield exist.swap match {
case Right(_) => AddResult.Success
case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((_, true)) =>
AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((ex, _)) => AddResult.Failure(ex)
}
}
}

View File

@ -20,11 +20,12 @@ object FlywayMigrate {
}
logger.info(s"Using migration locations: $locations")
val fw = Flyway.configure().
cleanDisabled(true).
dataSource(jdbc.url.asString, jdbc.user, jdbc.password).
locations(locations: _*).
load()
val fw = Flyway
.configure()
.cleanDisabled(true)
.dataSource(jdbc.url.asString, jdbc.user, jdbc.password)
.locations(locations: _*)
.load()
fw.repair()
fw.migrate()

View File

@ -12,40 +12,39 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for {
raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll))
n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId)))
f <- Stream.emit(raOpt).
unNoneTerminate.
map(_.fileId.id).
flatMap(store.bitpeace.delete).
compile.last
f <- Stream
.emit(raOpt)
.unNoneTerminate
.map(_.fileId.id)
.flatMap(store.bitpeace.delete)
.compile
.last
} yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0)
}
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = {
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for {
n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emit(ra.fileId.id).
flatMap(store.bitpeace.delete).
compile.last
f <- Stream.emit(ra.fileId.id).flatMap(store.bitpeace.delete).compile.last
} yield n + f.map(_ => 1).getOrElse(0)
}
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = {
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
for {
ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll))
ns <- ras.traverse(deleteAttachment[F](store))
} yield ns.sum
}
def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = {
val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns
val IC = RItem.Columns
val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
val q = fr"SELECT" ++ MC.proposals
.prefix("m")
.f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll))
@ -55,15 +54,22 @@ object QAttachment {
} yield MetaProposalList.flatten(ml)
}
def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = {
def getAttachmentMeta(
attachId: Ident,
collective: Ident
): ConnectionIO[Option[RAttachmentMeta]] = {
val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns
val IC = RItem.Columns
val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective)
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
.prefix("i")
.is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id
.prefix("a")
.is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a").is(attachId), IC.cid.prefix("i").is(collective))
q.query[RAttachmentMeta].option
}

View File

@ -8,27 +8,35 @@ import docspell.store.records.{RAttachment, RItem, RTag, RTagItem}
object QCollective {
case class InsightData( incoming: Int
, outgoing: Int
, bytes: Long
, tags: Map[String, Int])
case class InsightData(incoming: Int, outgoing: Int, bytes: Long, tags: Map[String, Int])
def getInsights(coll: Ident): ConnectionIO[InsightData] = {
val IC = RItem.Columns
val AC = RAttachment.Columns
val TC = RTag.Columns
val RC = RTagItem.Columns
val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)).
query[Int].unique
val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)).
query[Int].unique
val q0 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.incoming))
).query[Int].unique
val q1 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique
val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++
fr"WHERE" ++ IC.cid.is(coll)
val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++
val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
) ++
fr"FROM" ++ RTagItem.table ++ fr"r" ++
fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++
fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++

View File

@ -15,14 +15,16 @@ import org.log4s._
object QItem {
private[this] val logger = getLogger
case class ItemData( item: RItem
, corrOrg: Option[ROrganization]
, corrPerson: Option[RPerson]
, concPerson: Option[RPerson]
, concEquip: Option[REquipment]
, inReplyTo: Option[IdRef]
, tags: Vector[RTag]
, attachments: Vector[(RAttachment, FileMeta)]) {
case class ItemData(
item: RItem,
corrOrg: Option[ROrganization],
corrPerson: Option[RPerson],
concPerson: Option[RPerson],
concEquip: Option[REquipment],
inReplyTo: Option[IdRef],
tags: Vector[RTag],
attachments: Vector[(RAttachment, FileMeta)]
) {
def filterCollective(coll: Ident): Option[ItemData] =
if (item.cid == coll) Some(this) else None
@ -37,14 +39,35 @@ object QItem {
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg.prefix("i").is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson.prefix("i").is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson.prefix("i").is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment.prefix("i").is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo.prefix("i").is(RItem.Columns.id.prefix("ref")) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
.prefix("i")
.is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.prefix("i")
.is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.prefix("i")
.is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val q = cq.query[(RItem, Option[ROrganization], Option[RPerson], Option[RPerson], Option[REquipment], Option[IdRef])].option
val q = cq
.query[
(
RItem,
Option[ROrganization],
Option[RPerson],
Option[RPerson],
Option[REquipment],
Option[IdRef]
)
]
.option
val attachs = RAttachment.findByItemWithMeta(id)
val tags = RTag.findByItem(id)
@ -56,35 +79,38 @@ object QItem {
} yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att))
}
case class ListItem(
id: Ident,
name: String,
state: ItemState,
date: Timestamp,
dueDate: Option[Timestamp],
source: String,
direction: Direction,
created: Timestamp,
fileCount: Int,
corrOrg: Option[IdRef],
corrPerson: Option[IdRef],
concPerson: Option[IdRef],
concEquip: Option[IdRef]
)
case class ListItem( id: Ident
, name: String
, state: ItemState
, date: Timestamp
, dueDate: Option[Timestamp]
, source: String
, direction: Direction
, created: Timestamp
, fileCount: Int
, corrOrg: Option[IdRef]
, corrPerson: Option[IdRef]
, concPerson: Option[IdRef]
, concEquip: Option[IdRef])
case class Query( collective: Ident
, name: Option[String]
, states: Seq[ItemState]
, direction: Option[Direction]
, corrPerson: Option[Ident]
, corrOrg: Option[Ident]
, concPerson: Option[Ident]
, concEquip: Option[Ident]
, tagsInclude: List[Ident]
, tagsExclude: List[Ident]
, dateFrom: Option[Timestamp]
, dateTo: Option[Timestamp]
, dueDateFrom: Option[Timestamp]
, dueDateTo: Option[Timestamp])
case class Query(
collective: Ident,
name: Option[String],
states: Seq[ItemState],
direction: Option[Direction],
corrPerson: Option[Ident],
corrOrg: Option[Ident],
concPerson: Option[Ident],
concEquip: Option[Ident],
tagsInclude: List[Ident],
tagsExclude: List[Ident],
dateFrom: Option[Timestamp],
dateTo: Option[Timestamp],
dueDateFrom: Option[Timestamp],
dueDateTo: Option[Timestamp]
)
def findItems(q: Query): Stream[ConnectionIO, ListItem] = {
val IC = RItem.Columns
@ -97,70 +123,93 @@ object QItem {
val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name)
val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name)
val finalCols = commas(IC.id.prefix("i").f
, IC.name.prefix("i").f
, IC.state.prefix("i").f
, coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f)
, IC.dueDate.prefix("i").f
, IC.source.prefix("i").f
, IC.incoming.prefix("i").f
, IC.created.prefix("i").f
, fr"COALESCE(a.num, 0)"
, OC.oid.prefix("o0").f
, OC.name.prefix("o0").f
, PC.pid.prefix("p0").f
, PC.name.prefix("p0").f
, PC.pid.prefix("p1").f
, PC.name.prefix("p1").f
, EC.eid.prefix("e1").f
, EC.name.prefix("e1").f
val finalCols = commas(
IC.id.prefix("i").f,
IC.name.prefix("i").f,
IC.state.prefix("i").f,
coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f),
IC.dueDate.prefix("i").f,
IC.source.prefix("i").f,
IC.incoming.prefix("i").f,
IC.created.prefix("i").f,
fr"COALESCE(a.num, 0)",
OC.oid.prefix("o0").f,
OC.name.prefix("o0").f,
PC.pid.prefix("p0").f,
PC.name.prefix("p0").f,
PC.pid.prefix("p1").f,
PC.name.prefix("p1").f,
EC.eid.prefix("e1").f,
EC.name.prefix("e1").f
)
val withItem = selectSimple(itemCols, RItem.table, IC.cid is q.collective)
val withPerson = selectSimple(personCols, RPerson.table, PC.cid is q.collective)
val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid is q.collective)
val withEquips = selectSimple(equipCols, REquipment.table, EC.cid is q.collective)
val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.collective))
val withPerson = selectSimple(personCols, RPerson.table, PC.cid.is(q.collective))
val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid.is(q.collective))
val withEquips = selectSimple(equipCols, REquipment.table, EC.cid.is(q.collective))
val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++
fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")"
val query = withCTE("items" -> withItem
, "persons" -> withPerson
, "orgs" -> withOrgs
, "equips" -> withEquips
, "attachs" -> withAttach) ++
val query = withCTE(
"items" -> withItem,
"persons" -> withPerson,
"orgs" -> withOrgs,
"equips" -> withEquips,
"attachs" -> withAttach
) ++
fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++
fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson
.prefix("i")
.is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg
.prefix("i")
.is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson
.prefix("i")
.is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
.prefix("i")
.is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude.map(tid =>
selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId is tid)).
map(f => sql"(" ++ f ++ sql") ")
val tagSelectsIncl = q.tagsInclude
.map(tid =>
selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId.is(tid))
)
.map(f => sql"(" ++ f ++ sql") ")
// exclusive tags are OR-ed
val tagSelectsExcl =
if (q.tagsExclude.isEmpty) Fragment.empty
else selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId isOneOf q.tagsExclude)
else
selectSimple(
List(RTagItem.Columns.itemId),
RTagItem.table,
RTagItem.Columns.tagId.isOneOf(q.tagsExclude)
)
val name = q.name.map(queryWildcard)
val cond = and(
IC.cid.prefix("i") is q.collective,
IC.state.prefix("i") isOneOf q.states,
IC.incoming.prefix("i") isOrDiscard q.direction,
IC.cid.prefix("i").is(q.collective),
IC.state.prefix("i").isOneOf(q.states),
IC.incoming.prefix("i").isOrDiscard(q.direction),
name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty),
RPerson.Columns.pid.prefix("p0") isOrDiscard q.corrPerson,
ROrganization.Columns.oid.prefix("o0") isOrDiscard q.corrOrg,
RPerson.Columns.pid.prefix("p1") isOrDiscard q.concPerson,
REquipment.Columns.eid.prefix("e1") isOrDiscard q.concEquip,
RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson),
ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg),
RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson),
REquipment.Columns.eid.prefix("e1").isOrDiscard(q.concEquip),
if (q.tagsInclude.isEmpty) Fragment.empty
else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
else
IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")",
if (q.tagsExclude.isEmpty) Fragment.empty
else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")",
q.dateFrom.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d").getOrElse(Fragment.empty),
q.dateTo.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d").getOrElse(Fragment.empty),
q.dateFrom
.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d")
.getOrElse(Fragment.empty),
q.dateTo
.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d")
.getOrElse(Fragment.empty),
q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty),
q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty)
)
@ -171,7 +220,6 @@ object QItem {
frag.query[ListItem].stream
}
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
for {
tn <- store.transact(RTagItem.deleteItemTags(itemId))
@ -183,7 +231,9 @@ object QItem {
val IC = RItem.Columns
val AC = RAttachment.Columns
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
q.query[RItem].to[Vector]

View File

@ -15,27 +15,36 @@ import org.log4s._
object QJob {
private[this] val logger = getLogger
def takeNextJob[F[_]: Effect](store: Store[F])(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = {
Stream.range(0, 10).
evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)).
evalTap({ x =>
if (x.isLeft) logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
def takeNextJob[F[_]: Effect](
store: Store[F]
)(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] =
Stream
.range(0, 10)
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
.evalTap({ x =>
if (x.isLeft)
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
else ().pure[F]
}).
find(_.isRight).
flatMap({
})
.find(_.isRight)
.flatMap({
case Right(job) =>
Stream.emit(job)
case Left(_) =>
Stream.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")).map(_ => None)
}).
compile.last.map(_.flatten)
}
Stream
.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
.map(_ => None)
})
.compile
.last
.map(_.flatten)
private def takeNextJob1[F[_]: Effect](store: Store[F])( priority: Ident => F[Priority]
, worker: Ident
, retryPause: Duration
, currentTry: Int): F[Either[Unit, Option[RJob]]] = {
private def takeNextJob1[F[_]: Effect](store: Store[F])(
priority: Ident => F[Priority],
worker: Ident,
retryPause: Duration,
currentTry: Int
): F[Either[Unit, Option[RJob]]] = {
//if this fails, we have to restart takeNextJob
def markJob(job: RJob): F[Either[Unit, RJob]] =
store.transact(for {
@ -51,7 +60,9 @@ object QJob {
_ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}")
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
job <- group.map(g => store.transact(selectNextJob(g, prio, retryPause, now))).getOrElse((None: Option[RJob]).pure[F])
job <- group
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F])
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
res <- job.traverse(j => markJob(j))
} yield res.map(_.map(_.some)).getOrElse {
@ -60,7 +71,11 @@ object QJob {
}
}
def selectNextGroup(worker: Ident, now: Timestamp, initialPause: Duration): ConnectionIO[Option[Ident]] = {
def selectNextGroup(
worker: Ident,
now: Timestamp,
initialPause: Duration
): ConnectionIO[Option[Ident]] = {
val JC = RJob.Columns
val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck
@ -72,21 +87,30 @@ object QJob {
val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++
fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}"
val stateCond = or(jstate is waiting, and(jstate is stuck, stuckTrigger ++ fr"< ${now.toMillis}"))
val stateCond =
or(jstate.is(waiting), and(jstate.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++
fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++
fr"WHERE" ++ and(uworker is worker, stateCond) ++
fr"WHERE" ++ and(uworker.is(worker), stateCond) ++
fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
fr"WHERE" ++ stateCond
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
union.query[Ident].to[List].map(_.headOption) // either one or two results, but may be empty if RJob table is empty
union
.query[Ident]
.to[List]
.map(_.headOption) // either one or two results, but may be empty if RJob table is empty
}
def selectNextJob(group: Ident, prio: Priority, initialPause: Duration, now: Timestamp): ConnectionIO[Option[RJob]] = {
def selectNextJob(
group: Ident,
prio: Priority,
initialPause: Duration,
now: Timestamp
): ConnectionIO[Option[RJob]] = {
val JC = RJob.Columns
val psort =
if (prio == Priority.High) JC.priority.desc
@ -94,9 +118,17 @@ object QJob {
val waiting: JobState = JobState.Waiting
val stuck: JobState = JobState.Stuck
val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries) ++ fr"* ${initialPause.millis}"
val sql = selectSimple(JC.all, RJob.table,
and(JC.group is group, or(JC.state is waiting, and(JC.state is stuck, stuckTrigger ++ fr"< ${now.toMillis}")))) ++
val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(
JC.retries
) ++ fr"* ${initialPause.millis}"
val sql = selectSimple(
JC.all,
RJob.table,
and(
JC.group.is(group),
or(JC.state.is(waiting), and(JC.state.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}"))
)
) ++
orderBy(JC.state.asc, psort, JC.submitted.asc) ++
fr"LIMIT 1"
@ -150,9 +182,8 @@ object QJob {
def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] =
store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max))
def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = {
def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] =
store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ())
}
def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] =
store.transact(RJob.findFromIds(ids))
@ -165,10 +196,17 @@ object QJob {
def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = {
val refDate = now.minusHours(24)
val sql = selectSimple(JC.all, RJob.table,
and(JC.group is collective,
or(and(JC.state.isOneOf(done.toSeq), JC.submitted isGt refDate)
, JC.state.isOneOf((running ++ waiting).toSeq))))
val sql = selectSimple(
JC.all,
RJob.table,
and(
JC.group.is(collective),
or(
and(JC.state.isOneOf(done.toSeq), JC.submitted.isGt(refDate)),
JC.state.isOneOf((running ++ waiting).toSeq)
)
)
)
(sql ++ orderBy(JC.submitted.desc)).query[RJob].stream
}

View File

@ -12,10 +12,12 @@ import org.log4s._
object QLogin {
private[this] val logger = getLogger
case class Data( account: AccountId
, password: Password
, collectiveState: CollectiveState
, userState: UserState)
case class Data(
account: AccountId,
password: Password,
collectiveState: CollectiveState,
userState: UserState
)
def findUser(acc: AccountId): ConnectionIO[Option[Data]] = {
val ucid = UC.cid.prefix("u")
@ -28,7 +30,8 @@ object QLogin {
val sql = selectSimple(
List(ucid, login, pass, cstate, ustate),
RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c",
and(ucid is ccid, login is acc.user, ucid is acc.collective))
and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective))
)
logger.trace(s"SQL : $sql")
sql.query[Data].option

View File

@ -12,16 +12,24 @@ import docspell.store.records._
object QOrganization {
def findOrgAndContact(coll: Ident, order: OC.type => Column): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = {
ROrganization.findAll(coll, order).
evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs)))
}
def findPersonAndContact(coll: Ident, order: PC.type => Column): Stream[ConnectionIO, (RPerson, Vector[RContact])] = {
RPerson.findAll(coll, order).
evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs)))
}
def findOrgAndContact(
coll: Ident,
order: OC.type => Column
): Stream[ConnectionIO, (ROrganization, Vector[RContact])] =
ROrganization
.findAll(coll, order)
.evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs)))
def findPersonAndContact(
coll: Ident,
order: PC.type => Column
): Stream[ConnectionIO, (RPerson, Vector[RContact])] =
RPerson.findAll(coll, order).evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs)))
def addOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def addOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- ROrganization.insert(org)
cs <- contacts.toList.traverse(RContact.insert)
@ -32,7 +40,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def addPerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def addPerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- RPerson.insert(person)
cs <- contacts.toList.traverse(RContact.insert)
@ -43,7 +55,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def updateOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def updateOrg[F[_]](
org: ROrganization,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- ROrganization.update(org)
d <- RContact.deleteOrg(org.oid)
@ -55,7 +71,11 @@ object QOrganization {
store => store.add(insert, exists)
}
def updatePerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = {
def updatePerson[F[_]](
person: RPerson,
contacts: Seq[RContact],
cid: Ident
): Store[F] => F[AddResult] = {
val insert = for {
n <- RPerson.update(person)
d <- RContact.deletePerson(person.pid)
@ -67,15 +87,14 @@ object QOrganization {
store => store.add(insert, exists)
}
def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = {
def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] =
for {
n0 <- RItem.removeCorrOrg(collective, orgId)
n1 <- RContact.deleteOrg(orgId)
n2 <- ROrganization.delete(orgId, collective)
} yield n0 + n1 + n2
}
def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = {
def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] =
for {
n0 <- RItem.removeCorrPerson(collective, personId)
n1 <- RItem.removeConcPerson(collective, personId)
@ -83,4 +102,3 @@ object QOrganization {
n3 <- RPerson.delete(personId, collective)
} yield n0 + n1 + n2 + n3
}
}

View File

@ -24,19 +24,26 @@ object JobQueue {
def apply[F[_]: Effect](store: Store[F]): Resource[F, JobQueue[F]] =
Resource.pure(new JobQueue[F] {
def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] =
def nextJob(
prio: Ident => F[Priority],
worker: Ident,
retryPause: Duration
): F[Option[RJob]] =
logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
def insert(job: RJob): F[Unit] =
store.transact(RJob.insert(job)).
flatMap({ n =>
if (n != 1) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
store
.transact(RJob.insert(job))
.flatMap({ n =>
if (n != 1)
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F]
})
def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList.traverse(j => insert(j).attempt).
map(_.foreach {
jobs.toList
.traverse(j => insert(j).attempt)
.map(_.foreach {
case Right(()) =>
case Left(ex) =>
logger.error(ex)("Could not insert job. Skipping it.")

View File

@ -7,14 +7,14 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RAttachment( id: Ident
, itemId: Ident
, fileId: Ident
, position: Int
, created: Timestamp
, name: Option[String]) {
}
case class RAttachment(
id: Ident,
itemId: Ident,
fileId: Ident,
position: Int,
created: Timestamp,
name: Option[String]
) {}
object RAttachment {
@ -32,25 +32,34 @@ object RAttachment {
import Columns._
def insert(v: RAttachment): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}").update.run
insertRow(
table,
all,
fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}"
).update.run
def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(all, table, id is attachId).query[RAttachment].option
selectSimple(all, table, id.is(attachId)).query[RAttachment].option
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = {
selectSimple(all.map(_.prefix("a")), table ++ fr"a," ++ RItem.table ++ fr"i", and(
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] =
selectSimple(
all.map(_.prefix("a")),
table ++ fr"a," ++ RItem.table ++ fr"i",
and(
fr"a.itemid = i.itemid",
id.prefix("a") is attachId,
RItem.Columns.cid.prefix("i") is collective
)).query[RAttachment].option
}
id.prefix("a").is(attachId),
RItem.Columns.cid.prefix("i").is(collective)
)
).query[RAttachment].option
def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] =
selectSimple(all, table, itemId is id).query[RAttachment].to[Vector]
selectSimple(all, table, itemId.is(id)).query[RAttachment].to[Vector]
def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = {
val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id.prefix("i").is(itemId.prefix("a")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id
.prefix("i")
.is(itemId.prefix("a")) ++
fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll))
q.query[RAttachment].to[Vector]
}
@ -65,7 +74,7 @@ object RAttachment {
def delete(attachId: Ident): ConnectionIO[Int] =
for {
n0 <- RAttachmentMeta.delete(attachId)
n1 <- deleteFrom(table, id is attachId).update.run
n1 <- deleteFrom(table, id.is(attachId)).update.run
} yield n0 + n1
}

View File

@ -7,12 +7,12 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RAttachmentMeta(id: Ident
, content: Option[String]
, nerlabels: List[NerLabel]
, proposals: MetaProposalList) {
}
case class RAttachmentMeta(
id: Ident,
content: Option[String],
nerlabels: List[NerLabel],
proposals: MetaProposalList
) {}
object RAttachmentMeta {
def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty)
@ -32,7 +32,7 @@ object RAttachmentMeta {
insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run
def exists(attachId: Ident): ConnectionIO[Boolean] =
selectCount(id, table, id is attachId).query[Int].unique.map(_ > 0)
selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0)
def upsert(v: RAttachmentMeta): ConnectionIO[Int] =
for {
@ -41,22 +41,34 @@ object RAttachmentMeta {
} yield n1
def update(v: RAttachmentMeta): ConnectionIO[Int] =
updateRow(table, id is v.id, commas(
content setTo v.content,
nerlabels setTo v.nerlabels,
proposals setTo v.proposals
)).update.run
updateRow(
table,
id.is(v.id),
commas(
content.setTo(v.content),
nerlabels.setTo(v.nerlabels),
proposals.setTo(v.proposals)
)
).update.run
def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] =
updateRow(table, id is mid, commas(
nerlabels setTo labels
)).update.run
updateRow(
table,
id.is(mid),
commas(
nerlabels.setTo(labels)
)
).update.run
def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] =
updateRow(table, id is mid, commas(
proposals setTo plist
)).update.run
updateRow(
table,
id.is(mid),
commas(
proposals.setTo(plist)
)
).update.run
def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id is attachId).update.run
deleteFrom(table, id.is(attachId)).update.run
}

View File

@ -7,10 +7,7 @@ import doobie._
import doobie.implicits._
import fs2.Stream
case class RCollective( id: Ident
, state: CollectiveState
, language: Language
, created: Timestamp)
case class RCollective(id: Ident, state: CollectiveState, language: Language, created: Timestamp)
object RCollective {
@ -29,30 +26,38 @@ object RCollective {
import Columns._
def insert(value: RCollective): ConnectionIO[Int] = {
val sql = insertRow(table, Columns.all, fr"${value.id},${value.state},${value.language},${value.created}")
val sql = insertRow(
table,
Columns.all,
fr"${value.id},${value.state},${value.language},${value.created}"
)
sql.update.run
}
def update(value: RCollective): ConnectionIO[Int] = {
val sql = updateRow(table, id is value.id, commas(
state setTo value.state
))
val sql = updateRow(
table,
id.is(value.id),
commas(
state.setTo(value.state)
)
)
sql.update.run
}
def findLanguage(cid: Ident): ConnectionIO[Option[Language]] =
selectSimple(List(language), table, id is cid).query[Option[Language]].unique
selectSimple(List(language), table, id.is(cid)).query[Option[Language]].unique
def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] =
updateRow(table, id is cid, language setTo lang).update.run
updateRow(table, id.is(cid), language.setTo(lang)).update.run
def findById(cid: Ident): ConnectionIO[Option[RCollective]] = {
val sql = selectSimple(all, table, id is cid)
val sql = selectSimple(all, table, id.is(cid))
sql.query[RCollective].option
}
def existsById(cid: Ident): ConnectionIO[Boolean] = {
val sql = selectCount(id, table, id is cid)
val sql = selectCount(id, table, id.is(cid))
sql.query[Int].unique.map(_ > 0)
}

View File

@ -6,14 +6,13 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RContact(
contactId: Ident
, value: String
, kind: ContactKind
, personId: Option[Ident]
, orgId: Option[Ident]
, created: Timestamp) {
}
contactId: Ident,
value: String,
kind: ContactKind,
personId: Option[Ident],
orgId: Option[Ident],
created: Timestamp
) {}
object RContact {
@ -32,42 +31,49 @@ object RContact {
import Columns._
def insert(v: RContact): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}")
val sql = insertRow(
table,
all,
fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}"
)
sql.update.run
}
def update(v: RContact): ConnectionIO[Int] = {
val sql = updateRow(table, contactId is v.contactId, commas(
value setTo v.value,
kind setTo v.kind,
personId setTo v.personId,
orgId setTo v.orgId
))
val sql = updateRow(
table,
contactId.is(v.contactId),
commas(
value.setTo(v.value),
kind.setTo(v.kind),
personId.setTo(v.personId),
orgId.setTo(v.orgId)
)
)
sql.update.run
}
def delete(v: RContact): ConnectionIO[Int] =
deleteFrom(table, contactId is v.contactId).update.run
deleteFrom(table, contactId.is(v.contactId)).update.run
def deleteOrg(oid: Ident): ConnectionIO[Int] =
deleteFrom(table, orgId is oid).update.run
deleteFrom(table, orgId.is(oid)).update.run
def deletePerson(pid: Ident): ConnectionIO[Int] =
deleteFrom(table, personId is pid).update.run
deleteFrom(table, personId.is(pid)).update.run
def findById(id: Ident): ConnectionIO[Option[RContact]] = {
val sql = selectSimple(all, table, contactId is id)
val sql = selectSimple(all, table, contactId.is(id))
sql.query[RContact].option
}
def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, personId is pid)
val sql = selectSimple(all, table, personId.is(pid))
sql.query[RContact].to[Vector]
}
def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = {
val sql = selectSimple(all, table, orgId is oid)
val sql = selectSimple(all, table, orgId.is(oid))
sql.query[RContact].to[Vector]
}
}

View File

@ -5,13 +5,7 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class REquipment(
eid: Ident
, cid: Ident
, name: String
, created: Timestamp) {
}
case class REquipment(eid: Ident, cid: Ident, name: String, created: Timestamp) {}
object REquipment {
@ -27,39 +21,42 @@ object REquipment {
import Columns._
def insert(v: REquipment): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.eid},${v.cid},${v.name},${v.created}")
val sql = insertRow(table, all, fr"${v.eid},${v.cid},${v.name},${v.created}")
sql.update.run
}
def update(v: REquipment): ConnectionIO[Int] = {
val sql = updateRow(table, and(eid is v.eid, cid is v.cid), commas(
cid setTo v.cid,
name setTo v.name
))
val sql = updateRow(
table,
and(eid.is(v.eid), cid.is(v.cid)),
commas(
cid.setTo(v.cid),
name.setTo(v.name)
)
)
sql.update.run
}
def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = {
val sql = selectCount(eid, table, and(cid is coll, name is ename))
val sql = selectCount(eid, table, and(cid.is(coll), name.is(ename)))
sql.query[Int].unique.map(_ > 0)
}
def findById(id: Ident): ConnectionIO[Option[REquipment]] = {
val sql = selectSimple(all, table, eid is id)
val sql = selectSimple(all, table, eid.is(id))
sql.query[REquipment].option
}
def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[REquipment].to[Vector]
}
def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] =
selectSimple(List(eid, name), table, and(cid is coll,
name.lowerLike(equipName))).
query[IdRef].to[Vector]
selectSimple(List(eid, name), table, and(cid.is(coll), name.lowerLike(equipName)))
.query[IdRef]
.to[Vector]
def delete(id: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(eid is id, cid is coll)).update.run
deleteFrom(table, and(eid.is(id), cid.is(coll))).update.run
}

View File

@ -8,9 +8,7 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RInvitation(id: Ident, created: Timestamp) {
}
case class RInvitation(id: Ident, created: Timestamp) {}
object RInvitation {
@ -33,18 +31,16 @@ object RInvitation {
insertRow(table, all, fr"${v.id},${v.created}").update.run
def insertNew: ConnectionIO[RInvitation] =
generate[ConnectionIO].
flatMap(v => insert(v).map(_ => v))
generate[ConnectionIO].flatMap(v => insert(v).map(_ => v))
def findById(invite: Ident): ConnectionIO[Option[RInvitation]] =
selectSimple(all, table, id is invite).query[RInvitation].option
selectSimple(all, table, id.is(invite)).query[RInvitation].option
def delete(invite: Ident): ConnectionIO[Int] =
deleteFrom(table, id is invite).update.run
deleteFrom(table, id.is(invite)).update.run
def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = {
val get = selectCount(id, table, and(id is invite, created isGt minCreated)).
query[Int].unique
val get = selectCount(id, table, and(id.is(invite), created.isGt(minCreated))).query[Int].unique
for {
inv <- get
_ <- delete(invite)

View File

@ -8,32 +8,55 @@ import docspell.common._
import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RItem( id: Ident
, cid: Ident
, name: String
, itemDate: Option[Timestamp]
, source: String
, direction: Direction
, state: ItemState
, corrOrg: Option[Ident]
, corrPerson: Option[Ident]
, concPerson: Option[Ident]
, concEquipment: Option[Ident]
, inReplyTo: Option[Ident]
, dueDate: Option[Timestamp]
, created: Timestamp
, updated: Timestamp
, notes: Option[String]) {
}
case class RItem(
id: Ident,
cid: Ident,
name: String,
itemDate: Option[Timestamp],
source: String,
direction: Direction,
state: ItemState,
corrOrg: Option[Ident],
corrPerson: Option[Ident],
concPerson: Option[Ident],
concEquipment: Option[Ident],
inReplyTo: Option[Ident],
dueDate: Option[Timestamp],
created: Timestamp,
updated: Timestamp,
notes: Option[String]
) {}
object RItem {
def newItem[F[_]: Sync](cid: Ident, name: String, source: String, direction: Direction, state: ItemState): F[RItem] =
def newItem[F[_]: Sync](
cid: Ident,
name: String,
source: String,
direction: Direction,
state: ItemState
): F[RItem] =
for {
now <- Timestamp.current[F]
id <- Ident.randomId[F]
} yield RItem(id, cid, name, None, source, direction, state, None, None, None, None, None, None, now, now, None)
} yield RItem(
id,
cid,
name,
None,
source,
direction,
state,
None,
None,
None,
None,
None,
None,
now,
now,
None
)
val table = fr"item"
@ -54,110 +77,189 @@ object RItem {
val created = Column("created")
val updated = Column("updated")
val notes = Column("notes")
val all = List(id, cid, name, itemDate, source, incoming, state, corrOrg,
corrPerson, concPerson, concEquipment, inReplyTo, dueDate, created, updated, notes)
val all = List(
id,
cid,
name,
itemDate,
source,
incoming,
state,
corrOrg,
corrPerson,
concPerson,
concEquipment,
inReplyTo,
dueDate,
created,
updated,
notes
)
}
import Columns._
def insert(v: RItem): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++
insertRow(
table,
all,
fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++
fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++
fr"${v.created},${v.updated},${v.notes}").update.run
fr"${v.created},${v.updated},${v.notes}"
).update.run
def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] =
selectSimple(List(cid), table, id is itemId).query[Ident].option
selectSimple(List(cid), table, id.is(itemId)).query[Ident].option
def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, id is itemId, commas(state setTo itemState, updated setTo t)).update.run
n <- updateRow(table, id.is(itemId), commas(state.setTo(itemState), updated.setTo(t))).update.run
} yield n
def updateStateForCollective(itemId: Ident, itemState: ItemState, coll: Ident): ConnectionIO[Int] =
def updateStateForCollective(
itemId: Ident,
itemState: ItemState,
coll: Ident
): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(state setTo itemState, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t))
).update.run
} yield n
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(incoming setTo dir, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t))
).update.run
} yield n
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrOrg setTo org, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t))
).update.run
} yield n
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, corrOrg is Some(currentOrg)), commas(corrOrg setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(corrPerson setTo person, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, corrPerson is Some(currentPerson)), commas(corrPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concPerson setTo person, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, concPerson is Some(currentPerson)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(concEquipment setTo equip, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run
} yield n
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(cid is coll, concEquipment is Some(currentEquip)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run
n <- updateRow(
table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(notes setTo text, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t))
).update.run
} yield n
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(name setTo itemName, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t))
).update.run
} yield n
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(itemDate setTo date, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(table, and(id is itemId, cid is coll), commas(dueDate setTo date, updated setTo t)).update.run
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(id is itemId, cid is coll)).update.run
deleteFrom(table, and(id.is(itemId), cid.is(coll))).update.run
}

View File

@ -8,21 +8,23 @@ import docspell.store.impl.Column
import docspell.store.impl.Implicits._
import io.circe.Encoder
case class RJob(id: Ident
, task: Ident
, group: Ident
, args: String
, subject: String
, submitted: Timestamp
, submitter: Ident
, priority: Priority
, state: JobState
, retries: Int
, progress: Int
, tracker: Option[Ident]
, worker: Option[Ident]
, started: Option[Timestamp]
, finished: Option[Timestamp]) {
case class RJob(
id: Ident,
task: Ident,
group: Ident,
args: String,
subject: String,
submitted: Timestamp,
submitter: Ident,
priority: Priority,
state: JobState,
retries: Int,
progress: Int,
tracker: Option[Ident],
worker: Option[Ident],
started: Option[Timestamp],
finished: Option[Timestamp]
) {
def info: String =
s"${id.id.substring(0, 9)}.../${group.id}/${task.id}/$priority"
@ -30,16 +32,34 @@ case class RJob(id: Ident
object RJob {
def newJob[A](id: Ident
, task: Ident
, group: Ident
, args: A
, subject: String
, submitted: Timestamp
, submitter: Ident
, priority: Priority
, tracker: Option[Ident])(implicit E: Encoder[A]): RJob =
RJob(id, task, group, E(args).noSpaces, subject, submitted, submitter, priority, JobState.Waiting, 0, 0, tracker, None, None, None)
def newJob[A](
id: Ident,
task: Ident,
group: Ident,
args: A,
subject: String,
submitted: Timestamp,
submitter: Ident,
priority: Priority,
tracker: Option[Ident]
)(implicit E: Encoder[A]): RJob =
RJob(
id,
task,
group,
E(args).noSpaces,
subject,
submitted,
submitter,
priority,
JobState.Waiting,
0,
0,
tracker,
None,
None,
None
)
val table = fr"job"
@ -60,106 +80,160 @@ object RJob {
val started = Column("started")
val startedmillis = Column("startedmillis")
val finished = Column("finished")
val all = List(id,task,group,args,subject,submitted,submitter,priority,state,retries,progress,tracker,worker,started,finished)
val all = List(
id,
task,
group,
args,
subject,
submitted,
submitter,
priority,
state,
retries,
progress,
tracker,
worker,
started,
finished
)
}
import Columns._
def insert(v: RJob): ConnectionIO[Int] = {
val smillis = v.started.map(_.toMillis)
val sql = insertRow(table, all ++ List(startedmillis),
fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis")
val sql = insertRow(
table,
all ++ List(startedmillis),
fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis"
)
sql.update.run
}
def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] = {
def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] =
if (ids.isEmpty) Sync[ConnectionIO].pure(Vector.empty[RJob])
else selectSimple(all, table, id isOneOf ids).query[RJob].to[Vector]
}
else selectSimple(all, table, id.isOneOf(ids)).query[RJob].to[Vector]
def findByIdAndGroup(jobId: Ident, jobGroup: Ident): ConnectionIO[Option[RJob]] =
selectSimple(all, table, and(id is jobId, group is jobGroup)).query[RJob].option
selectSimple(all, table, and(id.is(jobId), group.is(jobGroup))).query[RJob].option
def setRunningToWaiting(workerId: Ident): ConnectionIO[Int] = {
val states: Seq[JobState] = List(JobState.Running, JobState.Scheduled)
updateRow(table, and(worker is workerId, state isOneOf states),
state setTo (JobState.Waiting: JobState)).update.run
updateRow(
table,
and(worker.is(workerId), state.isOneOf(states)),
state.setTo(JobState.Waiting: JobState)
).update.run
}
def incrementRetries(jobid: Ident): ConnectionIO[Int] =
updateRow(table, and(id is jobid, state is (JobState.Stuck: JobState)),
retries.f ++ fr"=" ++ retries.f ++ fr"+ 1").update.run
updateRow(
table,
and(id.is(jobid), state.is(JobState.Stuck: JobState)),
retries.f ++ fr"=" ++ retries.f ++ fr"+ 1"
).update.run
def setRunning(jobId: Ident, workerId: Ident, now: Timestamp): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Running: JobState),
started setTo now,
startedmillis setTo now.toMillis,
worker setTo workerId
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Running: JobState),
started.setTo(now),
startedmillis.setTo(now.toMillis),
worker.setTo(workerId)
)
).update.run
def setWaiting(jobId: Ident): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Waiting: JobState),
started setTo (None: Option[Timestamp]),
startedmillis setTo (None: Option[Long]),
finished setTo (None: Option[Timestamp])
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Waiting: JobState),
started.setTo(None: Option[Timestamp]),
startedmillis.setTo(None: Option[Long]),
finished.setTo(None: Option[Timestamp])
)
).update.run
def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] = {
def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] =
for {
_ <- incrementRetries(jobId)
n <- updateRow(table, and(id is jobId, or(worker.isNull, worker is workerId), state isOneOf Seq[JobState](JobState.Waiting, JobState.Stuck)), commas(
state setTo (JobState.Scheduled: JobState),
worker setTo workerId
)).update.run
n <- updateRow(
table,
and(
id.is(jobId),
or(worker.isNull, worker.is(workerId)),
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
),
commas(
state.setTo(JobState.Scheduled: JobState),
worker.setTo(workerId)
)
).update.run
} yield n
}
def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Success: JobState),
finished setTo now
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Success: JobState),
finished.setTo(now)
)
).update.run
def setStuck(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Stuck: JobState),
finished setTo now
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Stuck: JobState),
finished.setTo(now)
)
).update.run
def setFailed(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Failed: JobState),
finished setTo now
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Failed: JobState),
finished.setTo(now)
)
).update.run
def setCancelled(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
updateRow(table, id is jobId, commas(
state setTo (JobState.Cancelled: JobState),
finished setTo now
)).update.run
updateRow(
table,
id.is(jobId),
commas(
state.setTo(JobState.Cancelled: JobState),
finished.setTo(now)
)
).update.run
def getRetries(jobId: Ident): ConnectionIO[Option[Int]] =
selectSimple(List(retries), table, id is jobId).query[Int].option
selectSimple(List(retries), table, id.is(jobId)).query[Int].option
def setProgress(jobId: Ident, perc: Int): ConnectionIO[Int] =
updateRow(table, id is jobId, progress setTo perc).update.run
updateRow(table, id.is(jobId), progress.setTo(perc)).update.run
def selectWaiting: ConnectionIO[Option[RJob]] = {
val sql = selectSimple(all, table, state is (JobState.Waiting: JobState))
val sql = selectSimple(all, table, state.is(JobState.Waiting: JobState))
sql.query[RJob].to[Vector].map(_.headOption)
}
def selectGroupInState(states: Seq[JobState]): ConnectionIO[Vector[Ident]] = {
val sql = selectDistinct(List(group), table, state isOneOf states) ++ orderBy(group.f)
val sql = selectDistinct(List(group), table, state.isOneOf(states)) ++ orderBy(group.f)
sql.query[Ident].to[Vector]
}
def delete(jobId: Ident): ConnectionIO[Int] = {
def delete(jobId: Ident): ConnectionIO[Int] =
for {
n0 <- RJobLog.deleteAll(jobId)
n1 <- deleteFrom(table, id is jobId).update.run
n1 <- deleteFrom(table, id.is(jobId)).update.run
} yield n0 + n1
}
}

View File

@ -7,9 +7,7 @@ import docspell.common._
import docspell.store.impl.Column
import docspell.store.impl.Implicits._
case class RJobGroupUse(groupId: Ident, workerId: Ident) {
}
case class RJobGroupUse(groupId: Ident, workerId: Ident) {}
object RJobGroupUse {
@ -26,12 +24,11 @@ object RJobGroupUse {
insertRow(table, all, fr"${v.groupId},${v.workerId}").update.run
def updateGroup(v: RJobGroupUse): ConnectionIO[Int] =
updateRow(table, worker is v.workerId, group setTo v.groupId).update.run
updateRow(table, worker.is(v.workerId), group.setTo(v.groupId)).update.run
def setGroup(v: RJobGroupUse): ConnectionIO[Int] = {
def setGroup(v: RJobGroupUse): ConnectionIO[Int] =
updateGroup(v).flatMap(n => if (n > 0) n.pure[ConnectionIO] else insert(v))
}
def findGroup(workerId: Ident): ConnectionIO[Option[Ident]] =
selectSimple(List(group), table, worker is workerId).query[Ident].option
selectSimple(List(group), table, worker.is(workerId)).query[Ident].option
}

View File

@ -6,13 +6,7 @@ import docspell.common._
import docspell.store.impl.Column
import docspell.store.impl.Implicits._
case class RJobLog( id: Ident
, jobId: Ident
, level: LogLevel
, created: Timestamp
, message: String) {
}
case class RJobLog(id: Ident, jobId: Ident, level: LogLevel, created: Timestamp, message: String) {}
object RJobLog {
@ -32,8 +26,8 @@ object RJobLog {
insertRow(table, all, fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}").update.run
def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] =
(selectSimple(all, table, jobId is id) ++ orderBy(created.asc)).query[RJobLog].to[Vector]
(selectSimple(all, table, jobId.is(id)) ++ orderBy(created.asc)).query[RJobLog].to[Vector]
def deleteAll(job: Ident): ConnectionIO[Int] =
deleteFrom(table, jobId is job).update.run
deleteFrom(table, jobId.is(job)).update.run
}

View File

@ -8,10 +8,13 @@ import docspell.common._
import docspell.store.impl.Column
import docspell.store.impl.Implicits._
case class RNode(id: Ident, nodeType: NodeType, url: LenientUri, updated: Timestamp, created: Timestamp) {
}
case class RNode(
id: Ident,
nodeType: NodeType,
url: LenientUri,
updated: Timestamp,
created: Timestamp
) {}
object RNode {
@ -34,11 +37,15 @@ object RNode {
insertRow(table, all, fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}").update.run
def update(v: RNode): ConnectionIO[Int] =
updateRow(table, id is v.id, commas(
nodeType setTo v.nodeType,
url setTo v.url,
updated setTo v.updated
)).update.run
updateRow(
table,
id.is(v.id),
commas(
nodeType.setTo(v.nodeType),
url.setTo(v.url),
updated.setTo(v.updated)
)
).update.run
def set(v: RNode): ConnectionIO[Int] =
for {
@ -47,11 +54,11 @@ object RNode {
} yield n + k
def delete(appId: Ident): ConnectionIO[Int] =
(fr"DELETE FROM" ++ table ++ where(id is appId)).update.run
(fr"DELETE FROM" ++ table ++ where(id.is(appId))).update.run
def findAll(nt: NodeType): ConnectionIO[Vector[RNode]] =
selectSimple(all, table, nodeType is nt).query[RNode].to[Vector]
selectSimple(all, table, nodeType.is(nt)).query[RNode].to[Vector]
def findById(nodeId: Ident): ConnectionIO[Option[RNode]] =
selectSimple(all, table, id is nodeId).query[RNode].option
selectSimple(all, table, id.is(nodeId)).query[RNode].option
}

View File

@ -8,17 +8,16 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class ROrganization(
oid: Ident
, cid: Ident
, name: String
, street: String
, zip: String
, city: String
, country: String
, notes: Option[String]
, created: Timestamp) {
}
oid: Ident,
cid: Ident,
name: String,
street: String,
zip: String,
city: String,
country: String,
notes: Option[String],
created: Timestamp
) {}
object ROrganization {
@ -40,64 +39,77 @@ object ROrganization {
import Columns._
def insert(v: ROrganization): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created}")
val sql = insertRow(
table,
all,
fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created}"
)
sql.update.run
}
def update(v: ROrganization): ConnectionIO[Int] = {
val sql = updateRow(table, and(oid is v.oid, cid is v.cid), commas(
cid setTo v.cid,
name setTo v.name,
street setTo v.street,
zip setTo v.zip,
city setTo v.city,
country setTo v.country,
notes setTo v.notes
))
val sql = updateRow(
table,
and(oid.is(v.oid), cid.is(v.cid)),
commas(
cid.setTo(v.cid),
name.setTo(v.name),
street.setTo(v.street),
zip.setTo(v.zip),
city.setTo(v.city),
country.setTo(v.country),
notes.setTo(v.notes)
)
)
sql.update.run
}
def existsByName(coll: Ident, oname: String): ConnectionIO[Boolean] =
selectCount(oid, table, and(cid is coll, name is oname)).query[Int].unique.map(_ > 0)
selectCount(oid, table, and(cid.is(coll), name.is(oname))).query[Int].unique.map(_ > 0)
def findById(id: Ident): ConnectionIO[Option[ROrganization]] = {
val sql = selectSimple(all, table, cid is id)
val sql = selectSimple(all, table, cid.is(id))
sql.query[ROrganization].option
}
def find(coll: Ident, orgName: String): ConnectionIO[Option[ROrganization]] = {
val sql = selectSimple(all, table, and(cid is coll, name is orgName))
val sql = selectSimple(all, table, and(cid.is(coll), name.is(orgName)))
sql.query[ROrganization].option
}
def findLike(coll: Ident, orgName: String): ConnectionIO[Vector[IdRef]] =
selectSimple(List(oid, name), table, and(cid is coll,
name.lowerLike(orgName))).
query[IdRef].to[Vector]
selectSimple(List(oid, name), table, and(cid.is(coll), name.lowerLike(orgName)))
.query[IdRef]
.to[Vector]
def findLike(coll: Ident, contactKind: ContactKind, value: String): ConnectionIO[Vector[IdRef]] = {
def findLike(
coll: Ident,
contactKind: ContactKind,
value: String
): ConnectionIO[Vector[IdRef]] = {
val CC = RContact.Columns
val q = fr"SELECT DISTINCT" ++ commas(oid.prefix("o").f, name.prefix("o").f) ++
fr"FROM" ++ table ++ fr"o" ++
fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.orgId.prefix("c").is(oid.prefix("o")) ++
fr"WHERE" ++ and(cid.prefix("o").is(coll)
, CC.kind.prefix("c") is contactKind
, CC.value.prefix("c").lowerLike(value))
fr"WHERE" ++ and(
cid.prefix("o").is(coll),
CC.kind.prefix("c").is(contactKind),
CC.value.prefix("c").lowerLike(value)
)
q.query[IdRef].to[Vector]
}
def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, ROrganization] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[ROrganization].stream
}
def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = {
val sql = selectSimple(List(oid, name), table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(List(oid, name), table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[IdRef].to[Vector]
}
def delete(id: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(oid is id, cid is coll)).update.run
deleteFrom(table, and(oid.is(id), cid.is(coll))).update.run
}

View File

@ -8,18 +8,17 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RPerson(
pid: Ident
, cid: Ident
, name: String
, street: String
, zip: String
, city: String
, country: String
, notes: Option[String]
, concerning: Boolean
, created: Timestamp) {
}
pid: Ident,
cid: Ident,
name: String,
street: String,
zip: String,
city: String,
country: String,
notes: Option[String],
concerning: Boolean,
created: Timestamp
) {}
object RPerson {
@ -42,67 +41,86 @@ object RPerson {
import Columns._
def insert(v: RPerson): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created}")
val sql = insertRow(
table,
all,
fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created}"
)
sql.update.run
}
def update(v: RPerson): ConnectionIO[Int] = {
val sql = updateRow(table, and(pid is v.pid, cid is v.cid), commas(
cid setTo v.cid,
name setTo v.name,
street setTo v.street,
zip setTo v.zip,
city setTo v.city,
country setTo v.country,
concerning setTo v.concerning,
notes setTo v.notes
))
val sql = updateRow(
table,
and(pid.is(v.pid), cid.is(v.cid)),
commas(
cid.setTo(v.cid),
name.setTo(v.name),
street.setTo(v.street),
zip.setTo(v.zip),
city.setTo(v.city),
country.setTo(v.country),
concerning.setTo(v.concerning),
notes.setTo(v.notes)
)
)
sql.update.run
}
def existsByName(coll: Ident, pname: String): ConnectionIO[Boolean] =
selectCount(pid, table, and(cid is coll, name is pname)).query[Int].unique.map(_ > 0)
selectCount(pid, table, and(cid.is(coll), name.is(pname))).query[Int].unique.map(_ > 0)
def findById(id: Ident): ConnectionIO[Option[RPerson]] = {
val sql = selectSimple(all, table, cid is id)
val sql = selectSimple(all, table, cid.is(id))
sql.query[RPerson].option
}
def find(coll: Ident, personName: String): ConnectionIO[Option[RPerson]] = {
val sql = selectSimple(all, table, and(cid is coll, name is personName))
val sql = selectSimple(all, table, and(cid.is(coll), name.is(personName)))
sql.query[RPerson].option
}
def findLike(coll: Ident, personName: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] =
selectSimple(List(pid, name), table, and(cid is coll,
concerning is concerningOnly,
name.lowerLike(personName))).
query[IdRef].to[Vector]
def findLike(
coll: Ident,
personName: String,
concerningOnly: Boolean
): ConnectionIO[Vector[IdRef]] =
selectSimple(
List(pid, name),
table,
and(cid.is(coll), concerning.is(concerningOnly), name.lowerLike(personName))
).query[IdRef].to[Vector]
def findLike(coll: Ident, contactKind: ContactKind, value: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] = {
def findLike(
coll: Ident,
contactKind: ContactKind,
value: String,
concerningOnly: Boolean
): ConnectionIO[Vector[IdRef]] = {
val CC = RContact.Columns
val q = fr"SELECT DISTINCT" ++ commas(pid.prefix("p").f, name.prefix("p").f) ++
fr"FROM" ++ table ++ fr"p" ++
fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.personId.prefix("c").is(pid.prefix("p")) ++
fr"WHERE" ++ and(cid.prefix("p").is(coll)
, CC.kind.prefix("c") is contactKind
, concerning.prefix("p") is concerningOnly
, CC.value.prefix("c").lowerLike(value))
fr"WHERE" ++ and(
cid.prefix("p").is(coll),
CC.kind.prefix("c").is(contactKind),
concerning.prefix("p").is(concerningOnly),
CC.value.prefix("c").lowerLike(value)
)
q.query[IdRef].to[Vector]
}
def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, RPerson] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[RPerson].stream
}
def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = {
val sql = selectSimple(List(pid, name), table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(List(pid, name), table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[IdRef].to[Vector]
}
def delete(personId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(pid is personId, cid is coll)).update.run
deleteFrom(table, and(pid.is(personId), cid.is(coll))).update.run
}

View File

@ -7,16 +7,15 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RSource(
sid: Ident
, cid: Ident
, abbrev: String
, description: Option[String]
, counter: Int
, enabled: Boolean
, priority: Priority
, created: Timestamp) {
}
sid: Ident,
cid: Ident,
abbrev: String,
description: Option[String],
counter: Int,
enabled: Boolean,
priority: Priority,
created: Timestamp
) {}
object RSource {
@ -39,49 +38,59 @@ object RSource {
import Columns._
def insert(v: RSource): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}")
val sql = insertRow(
table,
all,
fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}"
)
sql.update.run
}
def updateNoCounter(v: RSource): ConnectionIO[Int] = {
val sql = updateRow(table, and(sid is v.sid, cid is v.cid), commas(
cid setTo v.cid,
abbrev setTo v.abbrev,
description setTo v.description,
enabled setTo v.enabled,
priority setTo v.priority
))
val sql = updateRow(
table,
and(sid.is(v.sid), cid.is(v.cid)),
commas(
cid.setTo(v.cid),
abbrev.setTo(v.abbrev),
description.setTo(v.description),
enabled.setTo(v.enabled),
priority.setTo(v.priority)
)
)
sql.update.run
}
def incrementCounter(source: String, coll: Ident): ConnectionIO[Int] =
updateRow(table, and(abbrev is source, cid is coll), counter.f ++ fr"=" ++ counter.f ++ fr"+ 1").update.run
updateRow(
table,
and(abbrev.is(source), cid.is(coll)),
counter.f ++ fr"=" ++ counter.f ++ fr"+ 1"
).update.run
def existsById(id: Ident): ConnectionIO[Boolean] = {
val sql = selectCount(sid, table, sid is id)
val sql = selectCount(sid, table, sid.is(id))
sql.query[Int].unique.map(_ > 0)
}
def existsByAbbrev(coll: Ident, abb: String): ConnectionIO[Boolean] = {
val sql = selectCount(sid, table, and(cid is coll, abbrev is abb))
val sql = selectCount(sid, table, and(cid.is(coll), abbrev.is(abb)))
sql.query[Int].unique.map(_ > 0)
}
def find(id: Ident): ConnectionIO[Option[RSource]] = {
val sql = selectSimple(all, table, sid is id)
val sql = selectSimple(all, table, sid.is(id))
sql.query[RSource].option
}
def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] =
selectSimple(List(cid), table, sid is sourceId).query[Ident].option
selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option
def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RSource]] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[RSource].to[Vector]
}
def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(sid is sourceId, cid is coll)).update.run
deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run
}

View File

@ -6,13 +6,12 @@ import docspell.store.impl._
import docspell.store.impl.Implicits._
case class RTag(
tagId: Ident
, collective: Ident
, name: String
, category: Option[String]
, created: Timestamp) {
}
tagId: Ident,
collective: Ident,
name: String,
category: Option[String],
created: Timestamp
) {}
object RTag {
@ -29,48 +28,60 @@ object RTag {
import Columns._
def insert(v: RTag): ConnectionIO[Int] = {
val sql = insertRow(table, all,
fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}")
val sql =
insertRow(table, all, fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}")
sql.update.run
}
def update(v: RTag): ConnectionIO[Int] = {
val sql = updateRow(table, and(tid is v.tagId, cid is v.collective), commas(
cid setTo v.collective,
name setTo v.name,
category setTo v.category
))
val sql = updateRow(
table,
and(tid.is(v.tagId), cid.is(v.collective)),
commas(
cid.setTo(v.collective),
name.setTo(v.name),
category.setTo(v.category)
)
)
sql.update.run
}
def findById(id: Ident): ConnectionIO[Option[RTag]] = {
val sql = selectSimple(all, table, tid is id)
val sql = selectSimple(all, table, tid.is(id))
sql.query[RTag].option
}
def findByIdAndCollective(id: Ident, coll: Ident): ConnectionIO[Option[RTag]] = {
val sql = selectSimple(all, table, and(tid is id, cid is coll))
val sql = selectSimple(all, table, and(tid.is(id), cid.is(coll)))
sql.query[RTag].option
}
def existsByName(tag: RTag): ConnectionIO[Boolean] = {
val sql = selectCount(tid, table, and(cid is tag.collective, name is tag.name, category is tag.category))
val sql = selectCount(
tid,
table,
and(cid.is(tag.collective), name.is(tag.name), category.is(tag.category))
)
sql.query[Int].unique.map(_ > 0)
}
def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RTag]] = {
val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f)
val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
sql.query[RTag].to[Vector]
}
def findByItem(itemId: Ident): ConnectionIO[Vector[RTag]] = {
val rcol = all.map(_.prefix("t"))
(selectSimple(rcol, table ++ fr"t," ++ RTagItem.table ++ fr"i", and(
RTagItem.Columns.itemId.prefix("i") is itemId,
(selectSimple(
rcol,
table ++ fr"t," ++ RTagItem.table ++ fr"i",
and(
RTagItem.Columns.itemId.prefix("i").is(itemId),
RTagItem.Columns.tagId.prefix("i").is(tid.prefix("t"))
)) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
)
) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
}
def delete(tagId: Ident, coll: Ident): ConnectionIO[Int] =
deleteFrom(table, and(tid is tagId, cid is coll)).update.run
deleteFrom(table, and(tid.is(tagId), cid.is(coll))).update.run
}

Some files were not shown because too many files have changed in this diff Show More