Apply scalafmt to all files

This commit is contained in:
Eike Kettner 2019-12-30 21:44:13 +01:00
parent 57e274e2b0
commit fc3e22e399
133 changed files with 3003 additions and 2112 deletions

View File

@ -27,38 +27,44 @@ trait BackendApp[F[_]] {
object BackendApp { object BackendApp {
def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] = def create[F[_]: ConcurrentEffect](
cfg: Config,
store: Store[F],
httpClientEc: ExecutionContext
): Resource[F, BackendApp[F]] =
for { for {
queue <- JobQueue(store) queue <- JobQueue(store)
loginImpl <- Login[F](store) loginImpl <- Login[F](store)
signupImpl <- OSignup[F](store) signupImpl <- OSignup[F](store)
collImpl <- OCollective[F](store) collImpl <- OCollective[F](store)
sourceImpl <- OSource[F](store) sourceImpl <- OSource[F](store)
tagImpl <- OTag[F](store) tagImpl <- OTag[F](store)
equipImpl <- OEquipment[F](store) equipImpl <- OEquipment[F](store)
orgImpl <- OOrganization(store) orgImpl <- OOrganization(store)
uploadImpl <- OUpload(store, queue, cfg, httpClientEc) uploadImpl <- OUpload(store, queue, cfg, httpClientEc)
nodeImpl <- ONode(store) nodeImpl <- ONode(store)
jobImpl <- OJob(store, httpClientEc) jobImpl <- OJob(store, httpClientEc)
itemImpl <- OItem(store) itemImpl <- OItem(store)
} yield new BackendApp[F] { } yield new BackendApp[F] {
val login: Login[F] = loginImpl val login: Login[F] = loginImpl
val signup: OSignup[F] = signupImpl val signup: OSignup[F] = signupImpl
val collective: OCollective[F] = collImpl val collective: OCollective[F] = collImpl
val source = sourceImpl val source = sourceImpl
val tag = tagImpl val tag = tagImpl
val equipment = equipImpl val equipment = equipImpl
val organization = orgImpl val organization = orgImpl
val upload = uploadImpl val upload = uploadImpl
val node = nodeImpl val node = nodeImpl
val job = jobImpl val job = jobImpl
val item = itemImpl val item = itemImpl
} }
def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config def apply[F[_]: ConcurrentEffect: ContextShift](
, connectEC: ExecutionContext cfg: Config,
, httpClientEc: ExecutionContext connectEC: ExecutionContext,
, blocker: Blocker): Resource[F, BackendApp[F]] = httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, BackendApp[F]] =
for { for {
store <- Store.create(cfg.jdbc, connectEC, blocker) store <- Store.create(cfg.jdbc, connectEC, blocker)
backend <- create(cfg, store, httpClientEc) backend <- create(cfg, store, httpClientEc)

View File

@ -4,13 +4,9 @@ import docspell.backend.signup.{Config => SignupConfig}
import docspell.common.MimeType import docspell.common.MimeType
import docspell.store.JdbcConfig import docspell.store.JdbcConfig
case class Config( jdbc: JdbcConfig case class Config(jdbc: JdbcConfig, signup: SignupConfig, files: Config.Files) {}
, signup: SignupConfig
, files: Config.Files) {
}
object Config { object Config {
case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType])
} }

View File

@ -50,14 +50,13 @@ object AuthToken {
Left("Invalid authenticator") Left("Invalid authenticator")
} }
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = { def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for { for {
salt <- Common.genSaltString[F] salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "") cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key) sig = sign(cd, key)
} yield cd.copy(sig = sig) } yield cd.copy(sig = sig)
}
private def sign(cd: AuthToken, key: ByteVector): String = { private def sign(cd: AuthToken, key: ByteVector): String = {
val raw = cd.millis.toString + cd.account.asString + cd.salt val raw = cd.millis.toString + cd.account.asString + cd.salt

View File

@ -45,45 +45,45 @@ object Login {
} }
def ok(session: AuthToken): Result = Ok(session) def ok(session: AuthToken): Result = Ok(session)
def invalidAuth: Result = InvalidAuth def invalidAuth: Result = InvalidAuth
def invalidTime: Result = InvalidTime def invalidTime: Result = InvalidTime
} }
def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] { def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] =
Resource.pure(new Login[F] {
def loginSession(config: Config)(sessionKey: String): F[Result] = def loginSession(config: Config)(sessionKey: String): F[Result] =
AuthToken.fromString(sessionKey) match { AuthToken.fromString(sessionKey) match {
case Right(at) => case Right(at) =>
if (at.sigInvalid(config.serverSecret)) Result.invalidAuth.pure[F] if (at.sigInvalid(config.serverSecret)) Result.invalidAuth.pure[F]
else if (at.isExpired(config.sessionValid)) Result.invalidTime.pure[F] else if (at.isExpired(config.sessionValid)) Result.invalidTime.pure[F]
else Result.ok(at).pure[F] else Result.ok(at).pure[F]
case Left(_) => case Left(_) =>
Result.invalidAuth.pure[F] Result.invalidAuth.pure[F]
}
def loginUserPass(config: Config)(up: UserPass): F[Result] =
AccountId.parse(up.user) match {
case Right(acc) =>
val okResult =
store.transact(RUser.updateLogin(acc)) *>
AuthToken.user(acc, config.serverSecret).map(Result.ok)
for {
data <- store.transact(QLogin.findUser(acc))
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
res <- if (data.exists(check(up.pass))) okResult
else Result.invalidAuth.pure[F]
} yield res
case Left(_) =>
Result.invalidAuth.pure[F]
}
private def check(given: String)(data: QLogin.Data): Boolean = {
val collOk = data.collectiveState == CollectiveState.Active ||
data.collectiveState == CollectiveState.ReadOnly
val userOk = data.userState == UserState.Active
val passOk = BCrypt.checkpw(given, data.password.pass)
collOk && userOk && passOk
} }
})
def loginUserPass(config: Config)(up: UserPass): F[Result] = {
AccountId.parse(up.user) match {
case Right(acc) =>
val okResult=
store.transact(RUser.updateLogin(acc)) *>
AuthToken.user(acc, config.serverSecret).map(Result.ok)
for {
data <- store.transact(QLogin.findUser(acc))
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
res <- if (data.exists(check(up.pass))) okResult
else Result.invalidAuth.pure[F]
} yield res
case Left(_) =>
Result.invalidAuth.pure[F]
}
}
private def check(given: String)(data: QLogin.Data): Boolean = {
val collOk = data.collectiveState == CollectiveState.Active ||
data.collectiveState == CollectiveState.ReadOnly
val userOk = data.userState == UserState.Active
val passOk = BCrypt.checkpw(given, data.password.pass)
collOk && userOk && passOk
}
})
} }

View File

@ -25,7 +25,11 @@ trait OCollective[F[_]] {
def insights(collective: Ident): F[InsightData] def insights(collective: Ident): F[InsightData]
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult]
} }
object OCollective { object OCollective {
@ -35,15 +39,15 @@ object OCollective {
sealed trait PassChangeResult sealed trait PassChangeResult
object PassChangeResult { object PassChangeResult {
case object UserNotFound extends PassChangeResult case object UserNotFound extends PassChangeResult
case object PasswordMismatch extends PassChangeResult case object PasswordMismatch extends PassChangeResult
case object UpdateFailed extends PassChangeResult case object UpdateFailed extends PassChangeResult
case object Success extends PassChangeResult case object Success extends PassChangeResult
def userNotFound: PassChangeResult = UserNotFound def userNotFound: PassChangeResult = UserNotFound
def passwordMismatch: PassChangeResult = PasswordMismatch def passwordMismatch: PassChangeResult = PasswordMismatch
def success: PassChangeResult = Success def success: PassChangeResult = Success
def updateFailed: PassChangeResult = UpdateFailed def updateFailed: PassChangeResult = UpdateFailed
} }
case class RegisterData(collName: Ident, login: Ident, password: Password, invite: Option[Ident]) case class RegisterData(collName: Ident, login: Ident, password: Password, invite: Option[Ident])
@ -63,39 +67,47 @@ object OCollective {
} }
} }
def apply[F[_]: Effect](store: Store[F]): Resource[F, OCollective[F]] =
def apply[F[_]:Effect](store: Store[F]): Resource[F, OCollective[F]] =
Resource.pure(new OCollective[F] { Resource.pure(new OCollective[F] {
def find(name: Ident): F[Option[RCollective]] = def find(name: Ident): F[Option[RCollective]] =
store.transact(RCollective.findById(name)) store.transact(RCollective.findById(name))
def updateLanguage(collective: Ident, lang: Language): F[AddResult] = def updateLanguage(collective: Ident, lang: Language): F[AddResult] =
store.transact(RCollective.updateLanguage(collective, lang)). store
attempt.map(AddResult.fromUpdate) .transact(RCollective.updateLanguage(collective, lang))
.attempt
.map(AddResult.fromUpdate)
def listUser(collective: Ident): F[Vector[RUser]] = { def listUser(collective: Ident): F[Vector[RUser]] =
store.transact(RUser.findAll(collective, _.login)) store.transact(RUser.findAll(collective, _.login))
}
def add(s: RUser): F[AddResult] = def add(s: RUser): F[AddResult] =
store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login)) store.add(
RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))),
RUser.exists(s.login)
)
def update(s: RUser): F[AddResult] = def update(s: RUser): F[AddResult] =
store.add(RUser.update(s), RUser.exists(s.login)) store.add(RUser.update(s), RUser.exists(s.login))
def deleteUser(login: Ident, collective: Ident): F[AddResult] = def deleteUser(login: Ident, collective: Ident): F[AddResult] =
store.transact(RUser.delete(login, collective)). store.transact(RUser.delete(login, collective)).attempt.map(AddResult.fromUpdate)
attempt.map(AddResult.fromUpdate)
def insights(collective: Ident): F[InsightData] = def insights(collective: Ident): F[InsightData] =
store.transact(QCollective.getInsights(collective)) store.transact(QCollective.getInsights(collective))
def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = { def changePassword(
accountId: AccountId,
current: Password,
newPass: Password
): F[PassChangeResult] = {
val q = for { val q = for {
optUser <- RUser.findByAccount(accountId) optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) n <- check
res = check match { .filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match {
case Some(true) => case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed
case Some(false) => case Some(false) =>

View File

@ -17,7 +17,6 @@ trait OEquipment[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult] def delete(id: Ident, collective: Ident): F[AddResult]
} }
object OEquipment { object OEquipment {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] =
@ -43,12 +42,10 @@ object OEquipment {
def delete(id: Ident, collective: Ident): F[AddResult] = { def delete(id: Ident, collective: Ident): F[AddResult] = {
val io = for { val io = for {
n0 <- RItem.removeConcEquip(collective, id) n0 <- RItem.removeConcEquip(collective, id)
n1 <- REquipment.delete(id, collective) n1 <- REquipment.delete(id, collective)
} yield n0 + n1 } yield n0 + n1
store.transact(io). store.transact(io).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
}) })
} }

View File

@ -62,90 +62,98 @@ object OItem {
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte]) case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] =
Resource.pure(new OItem[F] { Resource.pure(new OItem[F] {
def findItem(id: Ident, collective: Ident): F[Option[ItemData]] = def findItem(id: Ident, collective: Ident): F[Option[ItemData]] =
store.transact(QItem.findItem(id)). store.transact(QItem.findItem(id)).map(opt => opt.flatMap(_.filterCollective(collective)))
map(opt => opt.flatMap(_.filterCollective(collective)))
def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = { def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] =
store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector
}
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = { def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store.transact(RAttachment.findByIdAndCollective(id, collective)). store
flatMap({ .transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({
case Some(ra) => case Some(ra) =>
store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last. store.bitpeace
map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))))) .get(ra.fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m =>
AttachmentData[F](
ra,
m,
store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))
)
)
)
case None => case None =>
(None: Option[AttachmentData[F]]).pure[F] (None: Option[AttachmentData[F]]).pure[F]
}) })
}
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = { def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for { val db = for {
cid <- RItem.getCollective(item) cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO] nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO] else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
} yield nd + ni } yield nd + ni
store.transact(db). store.transact(db).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] = def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] =
store.transact(RItem.updateDirection(item, collective, direction)). store
attempt. .transact(RItem.updateDirection(item, collective, direction))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] = def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrOrg(item, collective, org)). store.transact(RItem.updateCorrOrg(item, collective, org)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateCorrPerson(item, collective, person)). store
attempt. .transact(RItem.updateCorrPerson(item, collective, person))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcPerson(item, collective, person)). store
attempt. .transact(RItem.updateConcPerson(item, collective, person))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] = def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] =
store.transact(RItem.updateConcEquip(item, collective, equip)). store
attempt. .transact(RItem.updateConcEquip(item, collective, equip))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] = def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] =
store.transact(RItem.updateNotes(item, collective, notes)). store.transact(RItem.updateNotes(item, collective, notes)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setName(item: Ident, name: String, collective: Ident): F[AddResult] = def setName(item: Ident, name: String, collective: Ident): F[AddResult] =
store.transact(RItem.updateName(item, collective, name)). store.transact(RItem.updateName(item, collective, name)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] = def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] =
store.transact(RItem.updateStateForCollective(item, state, collective)). store
attempt. .transact(RItem.updateStateForCollective(item, state, collective))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDate(item, collective, date)). store.transact(RItem.updateDate(item, collective, date)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] =
store.transact(RItem.updateDueDate(item, collective, date)). store
attempt. .transact(RItem.updateDueDate(item, collective, date))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
def delete(itemId: Ident, collective: Ident): F[Int] = def delete(itemId: Ident, collective: Ident): F[Int] =
QItem.delete(store)(itemId, collective) QItem.delete(store)(itemId, collective)

View File

@ -21,9 +21,9 @@ object OJob {
sealed trait JobCancelResult sealed trait JobCancelResult
object JobCancelResult { object JobCancelResult {
case object Removed extends JobCancelResult case object Removed extends JobCancelResult
case object CancelRequested extends JobCancelResult case object CancelRequested extends JobCancelResult
case object JobNotFound extends JobCancelResult case object JobNotFound extends JobCancelResult
} }
case class JobDetail(job: RJob, logs: Vector[RJobLog]) case class JobDetail(job: RJob, logs: Vector[RJobLog])
@ -36,15 +36,19 @@ object OJob {
jobs.filter(_.job.state == JobState.Running) jobs.filter(_.job.state == JobState.Running)
} }
def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] = def apply[F[_]: ConcurrentEffect](
store: Store[F],
clientEC: ExecutionContext
): Resource[F, OJob[F]] =
Resource.pure(new OJob[F] { Resource.pure(new OJob[F] {
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = { def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)). store
map(t => JobDetail(t._1, t._2)). .transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong))
compile.toVector. .map(t => JobDetail(t._1, t._2))
map(CollectiveQueueState) .compile
} .toVector
.map(CollectiveQueueState)
def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = { def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = {
def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] = def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] =
@ -58,26 +62,27 @@ object OJob {
val tryDelete = for { val tryDelete = for {
job <- RJob.findByIdAndGroup(id, collective) job <- RJob.findByIdAndGroup(id, collective)
jobm = job.filter(canDelete) jobm = job.filter(canDelete)
del <- jobm.traverse(j => RJob.delete(j.id)) del <- jobm.traverse(j => RJob.delete(j.id))
} yield del match { } yield del match {
case Some(_) => Right(JobCancelResult.Removed: JobCancelResult) case Some(_) => Right(JobCancelResult.Removed: JobCancelResult)
case None => Left(mustCancel(job)) case None => Left(mustCancel(job))
} }
def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] = def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] =
OJoex.cancelJob(job.id, worker, store, clientEC). OJoex
map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound) .cancelJob(job.id, worker, store, clientEC)
.map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound)
for { for {
tryDel <- store.transact(tryDelete) tryDel <- store.transact(tryDelete)
result <- tryDel match { result <- tryDel match {
case Right(r) => r.pure[F] case Right(r) => r.pure[F]
case Left(Some((job, worker))) => case Left(Some((job, worker))) =>
tryCancel(job, worker) tryCancel(job, worker)
case Left(None) => case Left(None) =>
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F] (JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
} }
} yield result } yield result
} }
}) })

View File

@ -13,24 +13,32 @@ import scala.concurrent.ExecutionContext
import org.log4s._ import org.log4s._
object OJoex { object OJoex {
private [this] val logger = getLogger private[this] val logger = getLogger
def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = { def notifyAll[F[_]: ConcurrentEffect](
store: Store[F],
clientExecutionContext: ExecutionContext
): F[Unit] =
for { for {
nodes <- store.transact(RNode.findAll(NodeType.Joex)) nodes <- store.transact(RNode.findAll(NodeType.Joex))
_ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext)) _ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext))
} yield () } yield ()
}
def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] = def cancelJob[F[_]: ConcurrentEffect](
jobId: Ident,
worker: Ident,
store: Store[F],
clientEc: ExecutionContext
): F[Boolean] =
for { for {
node <- store.transact(RNode.findById(worker)) node <- store.transact(RNode.findById(worker))
cancel <- node.traverse(joexCancel(clientEc)(_, jobId)) cancel <- node.traverse(joexCancel(clientEc)(_, jobId))
} yield cancel.getOrElse(false) } yield cancel.getOrElse(false)
private def joexCancel[F[_]: ConcurrentEffect](
private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = { ec: ExecutionContext
val notifyUrl = node.url/"api"/"v1"/"job"/job.id/"cancel" )(node: RNode, job: Ident): F[Boolean] = {
val notifyUrl = node.url / "api" / "v1" / "job" / job.id / "cancel"
BlazeClientBuilder[F](ec).resource.use { client => BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => true) client.expect[String](req).map(_ => true)
@ -38,7 +46,7 @@ object OJoex {
} }
private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = { private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = {
val notifyUrl = node.url/"api"/"v1"/"notify" val notifyUrl = node.url / "api" / "v1" / "notify"
val execute = BlazeClientBuilder[F](ec).resource.use { client => val execute = BlazeClientBuilder[F](ec).resource.use { client =>
val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString))
client.expect[String](req).map(_ => ()) client.expect[String](req).map(_ => ())

View File

@ -36,13 +36,15 @@ object OOrganization {
case class PersonAndContacts(person: RPerson, contacts: Seq[RContact]) case class PersonAndContacts(person: RPerson, contacts: Seq[RContact])
def apply[F[_] : Effect](store: Store[F]): Resource[F, OOrganization[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OOrganization[F]] =
Resource.pure(new OOrganization[F] { Resource.pure(new OOrganization[F] {
def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] = def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] =
store.transact(QOrganization.findOrgAndContact(account.collective, _.name)). store
map({ case (org, cont) => OrgAndContacts(org, cont) }). .transact(QOrganization.findOrgAndContact(account.collective, _.name))
compile.toVector .map({ case (org, cont) => OrgAndContacts(org, cont) })
.compile
.toVector
def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] = def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(ROrganization.findAllRef(account.collective, _.name)) store.transact(ROrganization.findAllRef(account.collective, _.name))
@ -54,9 +56,11 @@ object OOrganization {
QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store) QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store)
def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] = def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] =
store.transact(QOrganization.findPersonAndContact(account.collective, _.name)). store
map({ case (person, cont) => PersonAndContacts(person, cont) }). .transact(QOrganization.findPersonAndContact(account.collective, _.name))
compile.toVector .map({ case (person, cont) => PersonAndContacts(person, cont) })
.compile
.toVector
def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] = def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] =
store.transact(RPerson.findAllRef(account.collective, _.name)) store.transact(RPerson.findAllRef(account.collective, _.name))
@ -68,14 +72,13 @@ object OOrganization {
QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store) QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store)
def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] = def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deleteOrg(orgId, collective)). store.transact(QOrganization.deleteOrg(orgId, collective)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
def deletePerson(personId: Ident, collective: Ident): F[AddResult] = def deletePerson(personId: Ident, collective: Ident): F[AddResult] =
store.transact(QOrganization.deletePerson(personId, collective)). store
attempt. .transact(QOrganization.deletePerson(personId, collective))
map(AddResult.fromUpdate) .attempt
.map(AddResult.fromUpdate)
}) })
} }

View File

@ -41,8 +41,6 @@ object OSource {
} }
def delete(id: Ident, collective: Ident): F[AddResult] = def delete(id: Ident, collective: Ident): F[AddResult] =
store.transact(RSource.delete(id, collective)). store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
}) })
} }

View File

@ -17,7 +17,6 @@ trait OTag[F[_]] {
def delete(id: Ident, collective: Ident): F[AddResult] def delete(id: Ident, collective: Ident): F[AddResult]
} }
object OTag { object OTag {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] =
@ -47,10 +46,7 @@ object OTag {
n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId)) n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective)) n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective))
} yield n0.getOrElse(0) + n1.getOrElse(0) } yield n0.getOrElse(0) + n1.getOrElse(0)
store.transact(io). store.transact(io).attempt.map(AddResult.fromUpdate)
attempt.
map(AddResult.fromUpdate)
} }
}) })
} }

View File

@ -22,75 +22,113 @@ trait OUpload[F[_]] {
} }
object OUpload { object OUpload {
private [this] val logger = getLogger private[this] val logger = getLogger
case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte]) case class File[F[_]](
name: Option[String],
advertisedMime: Option[MimeType],
data: Stream[F, Byte]
)
case class UploadMeta( direction: Option[Direction] case class UploadMeta(
, sourceAbbrev: String direction: Option[Direction],
, validFileTypes: Seq[MimeType]) sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
case class UploadData[F[_]]( multiple: Boolean case class UploadData[F[_]](
, meta: UploadMeta multiple: Boolean,
, files: Vector[File[F]], priority: Priority, tracker: Option[Ident]) meta: UploadMeta,
files: Vector[File[F]],
priority: Priority,
tracker: Option[Ident]
)
sealed trait UploadResult sealed trait UploadResult
object UploadResult { object UploadResult {
case object Success extends UploadResult case object Success extends UploadResult
case object NoFiles extends UploadResult case object NoFiles extends UploadResult
case object NoSource extends UploadResult case object NoSource extends UploadResult
} }
def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] = def apply[F[_]: ConcurrentEffect](
store: Store[F],
queue: JobQueue[F],
cfg: Config,
httpClientEC: ExecutionContext
): Resource[F, OUpload[F]] =
Resource.pure(new OUpload[F] { Resource.pure(new OUpload[F] {
def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = { def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] =
for { for {
files <- data.files.traverse(saveFile).map(_.flatten) files <- data.files.traverse(saveFile).map(_.flatten)
pred <- checkFileList(files) pred <- checkFileList(files)
lang <- store.transact(RCollective.findLanguage(account.collective)) lang <- store.transact(RCollective.findLanguage(account.collective))
meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes) meta = ProcessItemArgs.ProcessMeta(
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList)) account.collective,
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker)) lang.getOrElse(Language.German),
_ <- logger.fdebug(s"Storing jobs: $job") data.meta.direction,
res <- job.traverse(submitJobs) data.meta.sourceAbbrev,
_ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)) data.meta.validFileTypes
)
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
else Vector(ProcessItemArgs(meta, files.toList))
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
_ <- logger.fdebug(s"Storing jobs: $job")
res <- job.traverse(submitJobs)
_ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective))
} yield res.fold(identity, identity) } yield res.fold(identity, identity)
}
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] = def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for { for {
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource)) sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev) abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev)) updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
accId = sOpt.map(source => AccountId(source.cid, source.sid)) accId = sOpt.map(source => AccountId(source.cid, source.sid))
result <- accId.traverse(acc => submit(updata, acc)) result <- accId.traverse(acc => submit(updata, acc))
} yield result.fold(identity, identity) } yield result.fold(identity, identity)
private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = { private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] =
for { for {
_ <- logger.fdebug(s"Storing jobs: $jobs") _ <- logger.fdebug(s"Storing jobs: $jobs")
_ <- queue.insertAll(jobs) _ <- queue.insertAll(jobs)
_ <- OJoex.notifyAll(store, httpClientEC) _ <- OJoex.notifyAll(store, httpClientEC)
} yield UploadResult.Success } yield UploadResult.Success
}
private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = { private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
logger.finfo(s"Receiving file $file") *> logger.finfo(s"Receiving file $file") *>
store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None). store.bitpeace
compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt. .saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None)
map(_.fold(ex => { .compile
logger.warn(ex)(s"Could not store file for processing!") .lastOrError
None .map(fm => Ident.unsafe(fm.id))
}, id => Some(ProcessItemArgs.File(file.name, id)))) .attempt
} .map(_.fold(ex => {
logger.warn(ex)(s"Could not store file for processing!")
None
}, id => Some(ProcessItemArgs.File(file.name, id))))
private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] = private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] =
Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(())) Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(()))
private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = { private def makeJobs(
args: Vector[ProcessItemArgs],
account: AccountId,
prio: Priority,
tracker: Option[Ident]
): F[Vector[RJob]] = {
def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob = def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob =
RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker) RJob.newJob(
id,
ProcessItemArgs.taskName,
account.collective,
arg,
arg.makeSubject,
now,
account.user,
prio,
tracker
)
for { for {
id <- Ident.randomId[F] id <- Ident.randomId[F]

View File

@ -20,10 +20,10 @@ object Config {
def fromString(str: String): Either[String, Mode] = def fromString(str: String): Either[String, Mode] =
str.toLowerCase match { str.toLowerCase match {
case "open" => Right(Open) case "open" => Right(Open)
case "invite" => Right(Invite) case "invite" => Right(Invite)
case "closed" => Right(Closed) case "closed" => Right(Closed)
case _ => Left(s"Invalid signup mode: $str") case _ => Left(s"Invalid signup mode: $str")
} }
def unsafe(str: String): Mode = def unsafe(str: String): Mode =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
@ -34,7 +34,7 @@ object Config {
Decoder.decodeString.emap(fromString) Decoder.decodeString.emap(fromString)
} }
def open: Mode = Mode.Open def open: Mode = Mode.Open
def invite: Mode = Mode.Invite def invite: Mode = Mode.Invite
def closed: Mode = Mode.Closed def closed: Mode = Mode.Closed

View File

@ -9,11 +9,11 @@ sealed trait NewInviteResult { self: Product =>
} }
object NewInviteResult { object NewInviteResult {
case class Success(id: Ident) extends NewInviteResult case class Success(id: Ident) extends NewInviteResult
case object InvitationDisabled extends NewInviteResult case object InvitationDisabled extends NewInviteResult
case object PasswordMismatch extends NewInviteResult case object PasswordMismatch extends NewInviteResult
def passwordMismatch: NewInviteResult = PasswordMismatch def passwordMismatch: NewInviteResult = PasswordMismatch
def invitationClosed: NewInviteResult = InvitationDisabled def invitationClosed: NewInviteResult = InvitationDisabled
def success(id: Ident): NewInviteResult = Success(id) def success(id: Ident): NewInviteResult = Success(id)
} }

View File

@ -21,19 +21,19 @@ trait OSignup[F[_]] {
object OSignup { object OSignup {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]:Effect](store: Store[F]): Resource[F, OSignup[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, OSignup[F]] =
Resource.pure(new OSignup[F] { Resource.pure(new OSignup[F] {
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = { def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
if (cfg.mode == Config.Mode.Invite) { if (cfg.mode == Config.Mode.Invite) {
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F] if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
NewInviteResult.passwordMismatch.pure[F]
else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id)) else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id))
} else { } else {
Effect[F].pure(NewInviteResult.invitationClosed) Effect[F].pure(NewInviteResult.invitationClosed)
} }
}
def register(cfg: Config)(data: RegisterData): F[SignupResult] = { def register(cfg: Config)(data: RegisterData): F[SignupResult] =
cfg.mode match { cfg.mode match {
case Config.Mode.Open => case Config.Mode.Open =>
addUser(data).map(SignupResult.fromAddResult) addUser(data).map(SignupResult.fromAddResult)
@ -45,11 +45,11 @@ object OSignup {
data.invite match { data.invite match {
case Some(inv) => case Some(inv) =>
for { for {
now <- Timestamp.current[F] now <- Timestamp.current[F]
min = now.minus(cfg.inviteTime) min = now.minus(cfg.inviteTime)
ok <- store.transact(RInvitation.useInvite(inv, min)) ok <- store.transact(RInvitation.useInvite(inv, min))
res <- if (ok) addUser(data).map(SignupResult.fromAddResult) res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
else SignupResult.invalidInvitationKey.pure[F] else SignupResult.invalidInvitationKey.pure[F]
_ <- if (retryInvite(res)) _ <- if (retryInvite(res))
logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
.transact( .transact(
@ -61,7 +61,6 @@ object OSignup {
SignupResult.invalidInvitationKey.pure[F] SignupResult.invalidInvitationKey.pure[F]
} }
} }
}
private def retryInvite(res: SignupResult): Boolean = private def retryInvite(res: SignupResult): Boolean =
res match { res match {
@ -77,29 +76,37 @@ object OSignup {
false false
} }
private def addUser(data: RegisterData): F[AddResult] = { private def addUser(data: RegisterData): F[AddResult] = {
def toRecords: F[(RCollective, RUser)] = def toRecords: F[(RCollective, RUser)] =
for { for {
id2 <- Ident.randomId[F] id2 <- Ident.randomId[F]
now <- Timestamp.current[F] now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now) c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now) u = RUser(
id2,
data.login,
data.collName,
PasswordCrypt.crypt(data.password),
UserState.Active,
None,
0,
None,
now
)
} yield (c, u) } yield (c, u)
def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = { def insert(coll: RCollective, user: RUser): ConnectionIO[Int] =
for { for {
n1 <- RCollective.insert(coll) n1 <- RCollective.insert(coll)
n2 <- RUser.insert(user) n2 <- RUser.insert(user)
} yield n1 + n2 } yield n1 + n2
}
def collectiveExists: ConnectionIO[Boolean] = def collectiveExists: ConnectionIO[Boolean] =
RCollective.existsById(data.collName) RCollective.existsById(data.collName)
val msg = s"The collective '${data.collName}' already exists." val msg = s"The collective '${data.collName}' already exists."
for { for {
cu <- toRecords cu <- toRecords
save <- store.add(insert(cu._1, cu._2), collectiveExists) save <- store.add(insert(cu._1, cu._2), collectiveExists)
} yield save.fold(identity, _.withMsg(msg), identity) } yield save.fold(identity, _.withMsg(msg), identity)
} }

View File

@ -2,27 +2,25 @@ package docspell.backend.signup
import docspell.store.AddResult import docspell.store.AddResult
sealed trait SignupResult { sealed trait SignupResult {}
}
object SignupResult { object SignupResult {
case object CollectiveExists extends SignupResult case object CollectiveExists extends SignupResult
case object InvalidInvitationKey extends SignupResult case object InvalidInvitationKey extends SignupResult
case object SignupClosed extends SignupResult case object SignupClosed extends SignupResult
case class Failure(ex: Throwable) extends SignupResult case class Failure(ex: Throwable) extends SignupResult
case object Success extends SignupResult case object Success extends SignupResult
def collectiveExists: SignupResult = CollectiveExists def collectiveExists: SignupResult = CollectiveExists
def invalidInvitationKey: SignupResult = InvalidInvitationKey def invalidInvitationKey: SignupResult = InvalidInvitationKey
def signupClosed: SignupResult = SignupClosed def signupClosed: SignupResult = SignupClosed
def failure(ex: Throwable): SignupResult = Failure(ex) def failure(ex: Throwable): SignupResult = Failure(ex)
def success: SignupResult = Success def success: SignupResult = Success
def fromAddResult(ar: AddResult): SignupResult = ar match { def fromAddResult(ar: AddResult): SignupResult = ar match {
case AddResult.Success => Success case AddResult.Success => Success
case AddResult.Failure(ex) => Failure(ex) case AddResult.Failure(ex) => Failure(ex)
case AddResult.EntityExists(_) => CollectiveExists case AddResult.EntityExists(_) => CollectiveExists
} }
} }

View File

@ -1,12 +1,14 @@
package docspell.common package docspell.common
case class Banner( component: String case class Banner(
, version: String component: String,
, gitHash: Option[String] version: String,
, jdbcUrl: LenientUri gitHash: Option[String],
, configFile: Option[String] jdbcUrl: LenientUri,
, appId: Ident configFile: Option[String],
, baseUrl: LenientUri) { appId: Ident,
baseUrl: LenientUri
) {
private val banner = private val banner =
"""______ _ _ """______ _ _
@ -17,16 +19,16 @@ case class Banner( component: String
||___/ \___/ \___|___/ .__/ \___|_|_| ||___/ \___/ \___|___/ .__/ \___|_|_|
| | | | | |
|""".stripMargin + |""".stripMargin +
s""" |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})""" s""" |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})"""
def render(prefix: String): String = { def render(prefix: String): String = {
val text = banner.split('\n').toList ++ List( val text = banner.split('\n').toList ++ List(
s"<< $component >>" s"<< $component >>",
, s"Id: ${appId.id}" s"Id: ${appId.id}",
, s"Base-Url: ${baseUrl.asString}" s"Base-Url: ${baseUrl.asString}",
, s"Database: ${jdbcUrl.asString}" s"Database: ${jdbcUrl.asString}",
, s"Config: ${configFile.getOrElse("")}" s"Config: ${configFile.getOrElse("")}",
, "" ""
) )
text.map(line => s"$prefix $line").mkString("\n") text.map(line => s"$prefix $line").mkString("\n")

View File

@ -21,7 +21,7 @@ object Direction {
str.toLowerCase match { str.toLowerCase match {
case "incoming" => Right(Incoming) case "incoming" => Right(Incoming)
case "outgoing" => Right(Outgoing) case "outgoing" => Right(Outgoing)
case _ => Left(s"No direction: $str") case _ => Left(s"No direction: $str")
} }
def unsafe(str: String): Direction = def unsafe(str: String): Direction =

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe._ import io.circe._
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
case class IdRef(id: Ident, name: String) { case class IdRef(id: Ident, name: String) {}
}
object IdRef { object IdRef {
@ -13,4 +11,4 @@ object IdRef {
deriveEncoder[IdRef] deriveEncoder[IdRef]
implicit val jsonDecoder: Decoder[IdRef] = implicit val jsonDecoder: Decoder[IdRef] =
deriveDecoder[IdRef] deriveDecoder[IdRef]
} }

View File

@ -10,18 +10,18 @@ sealed trait ItemState { self: Product =>
object ItemState { object ItemState {
case object Premature extends ItemState case object Premature extends ItemState
case object Processing extends ItemState case object Processing extends ItemState
case object Created extends ItemState case object Created extends ItemState
case object Confirmed extends ItemState case object Confirmed extends ItemState
def fromString(str: String): Either[String, ItemState] = def fromString(str: String): Either[String, ItemState] =
str.toLowerCase match { str.toLowerCase match {
case "premature" => Right(Premature) case "premature" => Right(Premature)
case "processing" => Right(Processing) case "processing" => Right(Processing)
case "created" => Right(Created) case "created" => Right(Created)
case "confirmed" => Right(Confirmed) case "confirmed" => Right(Confirmed)
case _ => Left(s"Invalid item state: $str") case _ => Left(s"Invalid item state: $str")
} }
def unsafe(str: String): ItemState = def unsafe(str: String): ItemState =
@ -32,4 +32,3 @@ object ItemState {
implicit val jsonEncoder: Encoder[ItemState] = implicit val jsonEncoder: Encoder[ItemState] =
Encoder.encodeString.contramap(_.name) Encoder.encodeString.contramap(_.name)
} }

View File

@ -31,14 +31,12 @@ object Language {
def fromString(str: String): Either[String, Language] = { def fromString(str: String): Either[String, Language] = {
val lang = str.toLowerCase val lang = str.toLowerCase
all.find(_.allNames.contains(lang)). all.find(_.allNames.contains(lang)).toRight(s"Unsupported or invalid language: $str")
toRight(s"Unsupported or invalid language: $str")
} }
def unsafe(str: String): Language = def unsafe(str: String): Language =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
implicit val jsonDecoder: Decoder[Language] = implicit val jsonDecoder: Decoder[Language] =
Decoder.decodeString.emap(fromString) Decoder.decodeString.emap(fromString)
implicit val jsonEncoder: Encoder[Language] = implicit val jsonEncoder: Encoder[Language] =

View File

@ -11,8 +11,8 @@ sealed trait LogLevel { self: Product =>
object LogLevel { object LogLevel {
case object Debug extends LogLevel { val toInt = 0 } case object Debug extends LogLevel { val toInt = 0 }
case object Info extends LogLevel { val toInt = 1 } case object Info extends LogLevel { val toInt = 1 }
case object Warn extends LogLevel { val toInt = 2 } case object Warn extends LogLevel { val toInt = 2 }
case object Error extends LogLevel { val toInt = 3 } case object Error extends LogLevel { val toInt = 3 }
def fromInt(n: Int): LogLevel = def fromInt(n: Int): LogLevel =
@ -26,12 +26,12 @@ object LogLevel {
def fromString(str: String): Either[String, LogLevel] = def fromString(str: String): Either[String, LogLevel] =
str.toLowerCase match { str.toLowerCase match {
case "debug" => Right(Debug) case "debug" => Right(Debug)
case "info" => Right(Info) case "info" => Right(Info)
case "warn" => Right(Warn) case "warn" => Right(Warn)
case "warning" => Right(Warn) case "warning" => Right(Warn)
case "error" => Right(Error) case "error" => Right(Error)
case _ => Left(s"Invalid log-level: $str") case _ => Left(s"Invalid log-level: $str")
} }
def unsafeString(str: String): LogLevel = def unsafeString(str: String): LogLevel =
@ -41,4 +41,4 @@ object LogLevel {
Decoder.decodeString.emap(fromString) Decoder.decodeString.emap(fromString)
implicit val jsonEncoder: Encoder[LogLevel] = implicit val jsonEncoder: Encoder[LogLevel] =
Encoder.encodeString.contramap(_.name) Encoder.encodeString.contramap(_.name)
} }

View File

@ -12,7 +12,7 @@ case class MimeType(primary: String, sub: String) {
def matches(other: MimeType): Boolean = def matches(other: MimeType): Boolean =
primary == other.primary && primary == other.primary &&
(sub == other.sub || sub == "*" ) (sub == other.sub || sub == "*")
} }
object MimeType { object MimeType {
@ -26,9 +26,10 @@ object MimeType {
def image(sub: String): MimeType = def image(sub: String): MimeType =
MimeType("image", partFromString(sub).throwLeft) MimeType("image", partFromString(sub).throwLeft)
private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet private[this] val validChars: Set[Char] =
(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
def parse(str: String): Either[String, MimeType] = { def parse(str: String): Either[String, MimeType] =
str.indexOf('/') match { str.indexOf('/') match {
case -1 => Left(s"Invalid MIME type: $str") case -1 => Left(s"Invalid MIME type: $str")
case n => case n =>
@ -37,7 +38,6 @@ object MimeType {
sub <- partFromString(str.substring(n + 1)) sub <- partFromString(str.substring(n + 1))
} yield MimeType(prim.toLowerCase, sub.toLowerCase) } yield MimeType(prim.toLowerCase, sub.toLowerCase)
} }
}
def unsafe(str: String): MimeType = def unsafe(str: String): MimeType =
parse(str).throwLeft parse(str).throwLeft
@ -47,12 +47,12 @@ object MimeType {
else Left(s"Invalid identifier: $s. Allowed chars: ${validChars.mkString}") else Left(s"Invalid identifier: $s. Allowed chars: ${validChars.mkString}")
val octetStream = application("octet-stream") val octetStream = application("octet-stream")
val pdf = application("pdf") val pdf = application("pdf")
val png = image("png") val png = image("png")
val jpeg = image("jpeg") val jpeg = image("jpeg")
val tiff = image("tiff") val tiff = image("tiff")
val html = text("html") val html = text("html")
val plain = text("plain") val plain = text("plain")
implicit val jsonEncoder: Encoder[MimeType] = implicit val jsonEncoder: Encoder[MimeType] =
Encoder.encodeString.contramap(_.asString) Encoder.encodeString.contramap(_.asString)

View File

@ -2,6 +2,4 @@ package docspell.common
import java.time.LocalDate import java.time.LocalDate
case class NerDateLabel(date: LocalDate, label: NerLabel) { case class NerDateLabel(date: LocalDate, label: NerLabel) {}
}

View File

@ -3,9 +3,7 @@ package docspell.common
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) { case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {}
}
object NerLabel { object NerLabel {
implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel] implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel]

View File

@ -10,13 +10,13 @@ sealed trait NodeType { self: Product =>
object NodeType { object NodeType {
case object Restserver extends NodeType case object Restserver extends NodeType
case object Joex extends NodeType case object Joex extends NodeType
def fromString(str: String): Either[String, NodeType] = def fromString(str: String): Either[String, NodeType] =
str.toLowerCase match { str.toLowerCase match {
case "restserver" => Right(Restserver) case "restserver" => Right(Restserver)
case "joex" => Right(Joex) case "joex" => Right(Joex)
case _ => Left(s"Invalid node type: $str") case _ => Left(s"Invalid node type: $str")
} }
def unsafe(str: String): NodeType = def unsafe(str: String): NodeType =

View File

@ -4,7 +4,7 @@ import io.circe.{Decoder, Encoder}
final class Password(val pass: String) extends AnyVal { final class Password(val pass: String) extends AnyVal {
def isEmpty: Boolean= pass.isEmpty def isEmpty: Boolean = pass.isEmpty
override def toString: String = override def toString: String =
if (pass.isEmpty) "<empty>" else "***" if (pass.isEmpty) "<empty>" else "***"

View File

@ -16,25 +16,23 @@ object Priority {
case object Low extends Priority case object Low extends Priority
def fromString(str: String): Either[String, Priority] = def fromString(str: String): Either[String, Priority] =
str.toLowerCase match { str.toLowerCase match {
case "high" => Right(High) case "high" => Right(High)
case "low" => Right(Low) case "low" => Right(Low)
case _ => Left(s"Invalid priority: $str") case _ => Left(s"Invalid priority: $str")
} }
def unsafe(str: String): Priority = def unsafe(str: String): Priority =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
def fromInt(n: Int): Priority = def fromInt(n: Int): Priority =
if (n <= toInt(Low)) Low if (n <= toInt(Low)) Low
else High else High
def toInt(p: Priority): Int = def toInt(p: Priority): Int =
p match { p match {
case Low => 0 case Low => 0
case High => 10 case High => 10
} }

View File

@ -6,14 +6,13 @@ import ProcessItemArgs._
case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
def makeSubject: String = { def makeSubject: String =
files.flatMap(_.name) match { files.flatMap(_.name) match {
case Nil => s"${meta.sourceAbbrev}: No files" case Nil => s"${meta.sourceAbbrev}: No files"
case n :: Nil => n case n :: Nil => n
case n1 :: n2 :: Nil => s"$n1, $n2" case n1 :: n2 :: Nil => s"$n1, $n2"
case _ => s"${files.size} files from ${meta.sourceAbbrev}" case _ => s"${files.size} files from ${meta.sourceAbbrev}"
} }
}
} }
@ -21,11 +20,13 @@ object ProcessItemArgs {
val taskName = Ident.unsafe("process-item") val taskName = Ident.unsafe("process-item")
case class ProcessMeta( collective: Ident case class ProcessMeta(
, language: Language collective: Ident,
, direction: Option[Direction] language: Language,
, sourceAbbrev: String direction: Option[Direction],
, validFileTypes: Seq[MimeType]) sourceAbbrev: String,
validFileTypes: Seq[MimeType]
)
object ProcessMeta { object ProcessMeta {
implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta] implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta]

View File

@ -30,9 +30,7 @@ object Timestamp {
def current[F[_]: Sync]: F[Timestamp] = def current[F[_]: Sync]: F[Timestamp] =
Sync[F].delay(Timestamp(Instant.now)) Sync[F].delay(Timestamp(Instant.now))
implicit val encodeTimestamp: Encoder[Timestamp] =
implicit val encodeTimestamp: Encoder[Timestamp] =
BaseJsonCodecs.encodeInstantEpoch.contramap(_.value) BaseJsonCodecs.encodeInstantEpoch.contramap(_.value)
implicit val decodeTimestamp: Decoder[Timestamp] = implicit val decodeTimestamp: Decoder[Timestamp] =

View File

@ -12,19 +12,18 @@ object UserState {
/** The user is blocked by an admin. */ /** The user is blocked by an admin. */
case object Disabled extends UserState case object Disabled extends UserState
def fromString(s: String): Either[String, UserState] = def fromString(s: String): Either[String, UserState] =
s.toLowerCase match { s.toLowerCase match {
case "active" => Right(Active) case "active" => Right(Active)
case "disabled" => Right(Disabled) case "disabled" => Right(Disabled)
case _ => Left(s"Not a state value: $s") case _ => Left(s"Not a state value: $s")
} }
def unsafe(str: String): UserState = def unsafe(str: String): UserState =
fromString(str).fold(sys.error, identity) fromString(str).fold(sys.error, identity)
def asString(s: UserState): String = s match { def asString(s: UserState): String = s match {
case Active => "active" case Active => "active"
case Disabled => "disabled" case Disabled => "disabled"
} }
@ -34,4 +33,4 @@ object UserState {
implicit val userStateDecoder: Decoder[UserState] = implicit val userStateDecoder: Decoder[UserState] =
Decoder.decodeString.emap(UserState.fromString) Decoder.decodeString.emap(UserState.fromString)
} }

View File

@ -4,18 +4,18 @@ trait EitherSyntax {
implicit final class LeftStringEitherOps[A](e: Either[String, A]) { implicit final class LeftStringEitherOps[A](e: Either[String, A]) {
def throwLeft: A = e match { def throwLeft: A = e match {
case Right(a) => a case Right(a) => a
case Left(err) => sys.error(err) case Left(err) => sys.error(err)
} }
} }
implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) { implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) {
def throwLeft: A = e match { def throwLeft: A = e match {
case Right(a) => a case Right(a) => a
case Left(err) => throw err case Left(err) => throw err
} }
} }
} }
object EitherSyntax extends EitherSyntax object EitherSyntax extends EitherSyntax

View File

@ -11,13 +11,18 @@ trait StreamSyntax {
implicit class StringStreamOps[F[_]](s: Stream[F, String]) { implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] = def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
s.fold("")(_ + _). s.fold("")(_ + _)
compile.last. .compile
map(optStr => for { .last
str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value")) .map(optStr =>
json <- parse(str).leftMap(_.underlying) for {
value <- json.as[A] str <- optStr
} yield value) .map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying)
value <- json.as[A]
} yield value
)
} }

View File

@ -5,17 +5,23 @@ import docspell.joex.scheduler.SchedulerConfig
import docspell.store.JdbcConfig import docspell.store.JdbcConfig
import docspell.text.ocr.{Config => OcrConfig} import docspell.text.ocr.{Config => OcrConfig}
case class Config(appId: Ident case class Config(
, baseUrl: LenientUri appId: Ident,
, bind: Config.Bind baseUrl: LenientUri,
, jdbc: JdbcConfig bind: Config.Bind,
, scheduler: SchedulerConfig jdbc: JdbcConfig,
, extraction: OcrConfig scheduler: SchedulerConfig,
extraction: OcrConfig
) )
object Config { object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") val postgres =
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
case class Bind(address: String, port: Int) case class Bind(address: String, port: Int)
} }

View File

@ -11,7 +11,6 @@ object ConfigFile {
def loadConfig: Config = def loadConfig: Config =
ConfigSource.default.at("docspell.joex").loadOrThrow[Config] ConfigSource.default.at("docspell.joex").loadOrThrow[Config]
object Implicits { object Implicits {
implicit val countingSchemeReader: ConfigReader[CountingScheme] = implicit val countingSchemeReader: ConfigReader[CountingScheme] =
ConfigReader[String].emap(reason(CountingScheme.readString)) ConfigReader[String].emap(reason(CountingScheme.readString))

View File

@ -12,11 +12,13 @@ import fs2.concurrent.SignallingRef
import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext
final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer](
, nodeOps: ONode[F] cfg: Config,
, store: Store[F] nodeOps: ONode[F],
, termSignal: SignallingRef[F, Boolean] store: Store[F],
, val scheduler: Scheduler[F]) extends JoexApp[F] { termSignal: SignallingRef[F, Boolean],
val scheduler: Scheduler[F]
) extends JoexApp[F] {
def init: F[Unit] = { def init: F[Unit] = {
val run = scheduler.start.compile.drain val run = scheduler.start.compile.drain
@ -40,17 +42,25 @@ final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Conf
object JoexAppImpl { object JoexAppImpl {
def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config def create[F[_]: ConcurrentEffect: ContextShift: Timer](
, termSignal: SignallingRef[F, Boolean] cfg: Config,
, connectEC: ExecutionContext termSignal: SignallingRef[F, Boolean],
, blocker: Blocker): Resource[F, JoexApp[F]] = connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, JoexApp[F]] =
for { for {
store <- Store.create(cfg.jdbc, connectEC, blocker) store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store) nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store). sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])). .withTask(
resource JobTask.json(
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) ProcessItemArgs.taskName,
appR <- Resource.make(app.init.map(_ => app))(_.shutdown) ItemHandler[F](cfg.extraction),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR } yield appR
} }

View File

@ -15,20 +15,26 @@ import scala.concurrent.ExecutionContext
object JoexServer { object JoexServer {
private case class App[F[_]](
httpApp: HttpApp[F],
termSig: SignallingRef[F, Boolean],
exitRef: Ref[F, ExitCode]
)
private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode]) def stream[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker) connectEC: ExecutionContext,
(implicit T: Timer[F]): Stream[F, Nothing] = { blocker: Blocker
)(implicit T: Timer[F]): Stream[F, Nothing] = {
val app = for { val app = for {
signal <- Resource.liftF(SignallingRef[F, Boolean](false)) signal <- Resource.liftF(SignallingRef[F, Boolean](false))
exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success)) exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success))
joexApp <- JoexAppImpl.create[F](cfg, signal, connectEC, blocker) joexApp <- JoexAppImpl.create[F](cfg, signal, connectEC, blocker)
httpApp = Router( httpApp = Router(
"/api/info" -> InfoRoutes(), "/api/info" -> InfoRoutes(),
"/api/v1" -> JoexRoutes(joexApp) "/api/v1" -> JoexRoutes(joexApp)
).orNotFound ).orNotFound
// With Middlewares in place // With Middlewares in place
@ -36,14 +42,15 @@ object JoexServer {
} yield App(finalHttpApp, signal, exitCode) } yield App(finalHttpApp, signal, exitCode)
Stream
Stream.resource(app).flatMap(app => .resource(app)
BlazeServerBuilder[F]. .flatMap(app =>
bindHttp(cfg.bind.port, cfg.bind.address). BlazeServerBuilder[F]
withHttpApp(app.httpApp). .bindHttp(cfg.bind.port, cfg.bind.address)
withoutBanner. .withHttpApp(app.httpApp)
serveWhile(app.termSig, app.exitRef) .withoutBanner
) .serveWhile(app.termSig, app.exitRef)
)
}.drain }.drain
} }

View File

@ -14,10 +14,12 @@ object Main extends IOApp {
private[this] val logger = getLogger private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor( val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))) Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc) val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService( val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))) Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))
)
def run(args: List[String]) = { def run(args: List[String]) = {
args match { args match {
@ -40,12 +42,15 @@ object Main extends IOApp {
} }
val cfg = ConfigFile.loadConfig val cfg = ConfigFile.loadConfig
val banner = Banner("JOEX" val banner = Banner(
, BuildInfo.version "JOEX",
, BuildInfo.gitHeadCommit BuildInfo.version,
, cfg.jdbc.url BuildInfo.gitHeadCommit,
, Option(System.getProperty("config.file")) cfg.jdbc.url,
, cfg.appId, cfg.baseUrl) Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}") logger.info(s"\n${banner.render("***>")}")
JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success) JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success)
} }

View File

@ -16,56 +16,76 @@ object CreateItem {
def apply[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] =
findExisting[F].flatMap { findExisting[F].flatMap {
case Some(ri) => Task.pure(ri) case Some(ri) => Task.pure(ri)
case None => createNew[F] case None => createNew[F]
} }
def createNew[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = def createNew[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet
def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files). def fileMetas(itemId: Ident, now: Timestamp) =
flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))). Stream
collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }). .emits(ctx.args.files)
zipWithIndex. .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
evalMap({ case (f, index) => .collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f })
Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)) .zipWithIndex
}). .evalMap({
compile.toVector case (f, index) =>
Ident
.randomId[F]
.map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name))
})
.compile
.toVector
val item = RItem.newItem[F](ctx.args.meta.collective val item = RItem.newItem[F](
, ctx.args.makeSubject ctx.args.meta.collective,
, ctx.args.meta.sourceAbbrev ctx.args.makeSubject,
, ctx.args.meta.direction.getOrElse(Direction.Incoming) ctx.args.meta.sourceAbbrev,
, ItemState.Premature) ctx.args.meta.direction.getOrElse(Direction.Incoming),
ItemState.Premature
)
for { for {
_ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)") _ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)")
time <- Duration.stopTime[F] time <- Duration.stopTime[F]
it <- item it <- item
n <- ctx.store.transact(RItem.insert(it)) n <- ctx.store.transact(RItem.insert(it))
_ <- if (n != 1) storeItemError[F](ctx) else ().pure[F] _ <- if (n != 1) storeItemError[F](ctx) else ().pure[F]
fm <- fileMetas(it.id, it.created) fm <- fileMetas(it.id, it.created)
k <- fm.traverse(a => ctx.store.transact(RAttachment.insert(a))) k <- fm.traverse(a => ctx.store.transact(RAttachment.insert(a)))
_ <- logDifferences(ctx, fm, k.sum) _ <- logDifferences(ctx, fm, k.sum)
dur <- time dur <- time
_ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}") _ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}")
} yield ItemData(it, fm, Vector.empty, Vector.empty) } yield ItemData(it, fm, Vector.empty, Vector.empty)
} }
def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
Task { ctx => Task { ctx =>
for { for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F] _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) else ().pure[F]
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F] ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))) _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
} yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)) else ().pure[F]
rms <- cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
} yield cand.headOption.map(ri =>
ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)
)
} }
private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] = private def logDifferences[F[_]: Sync](
ctx: Context[F, ProcessItemArgs],
saved: Vector[RAttachment],
saveCount: Int
): F[Unit] =
if (ctx.args.files.size != saved.size) { if (ctx.args.files.size != saved.size) {
ctx.logger.warn(s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount") ctx.logger.warn(
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
)
} else { } else {
().pure[F] ().pure[F]
} }

View File

@ -19,45 +19,65 @@ object FindProposal {
def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
val rmas = data.metas.map(rm => val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
rm.copy(nerlabels = removeDuplicates(rm.nerlabels)))
ctx.logger.info("Starting find-proposal") *> ctx.logger.info("Starting find-proposal") *>
rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))). rmas
flatMap(rmv => rmv.traverse(rm => .traverse(rm =>
ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *> processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))). )
map(_ => data.copy(metas = rmv))) .flatMap(rmv =>
rmv
.traverse(rm =>
ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *>
ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))
)
.map(_ => data.copy(metas = rmv))
)
} }
def processAttachment[F[_]: Sync]( rm: RAttachmentMeta def processAttachment[F[_]: Sync](
, rd: Vector[NerDateLabel] rm: RAttachmentMeta,
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { rd: Vector[NerDateLabel],
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx)) val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx))
List(finder.find(rm.nerlabels), makeDateProposal(rd)). List(finder.find(rm.nerlabels), makeDateProposal(rd))
traverse(identity).map(MetaProposalList.flatten) .traverse(identity)
.map(MetaProposalList.flatten)
} }
def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = { def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] =
Timestamp.current[F].map { now => Timestamp.current[F].map { now =>
val latestFirst = dates.sortWith(_.date isAfter _.date) val latestFirst = dates.sortWith((l1, l2) => l1.date.isAfter(l2.date))
val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate
val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate)) val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate))
val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate, val dueDates = MetaProposalList.fromSeq1(
after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) MetaProposalType.DueDate,
val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate, after.map(ndl =>
before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
val itemDates = MetaProposalList.fromSeq1(
MetaProposalType.DocDate,
before.map(ndl =>
Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label))
)
)
MetaProposalList.flatten(Seq(dueDates, itemDates)) MetaProposalList.flatten(Seq(dueDates, itemDates))
} }
}
def removeDuplicates(labels: List[NerLabel]): List[NerLabel] = def removeDuplicates(labels: List[NerLabel]): List[NerLabel] =
labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) => labels
if (seen.contains(el.tag.name+el.label.toLowerCase)) (seen, result) .foldLeft((Set.empty[String], List.empty[NerLabel])) {
else (seen + (el.tag.name + el.label.toLowerCase), el :: result) case ((seen, result), el) =>
}._2.sortBy(_.startPosition) if (seen.contains(el.tag.name + el.label.toLowerCase)) (seen, result)
else (seen + (el.tag.name + el.label.toLowerCase), el :: result)
}
._2
.sortBy(_.startPosition)
trait Finder[F[_]] { self => trait Finder[F[_]] { self =>
def find(labels: Seq[NerLabel]): F[MetaProposalList] def find(labels: Seq[NerLabel]): F[MetaProposalList]
@ -80,12 +100,14 @@ object FindProposal {
else f.map(ml1 => ml0.fillEmptyFrom(ml1)) else f.map(ml1 => ml0.fillEmptyFrom(ml1))
}) })
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*) def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
(implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] = implicit F: FlatMap[F],
flatMap(res0 => { F2: Applicative[F]
): Finder[F] =
flatMap { res0 =>
if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0) if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0)
else f.map(res1 => res0.fillEmptyFrom(res1)) else f.map(res1 => res0.fillEmptyFrom(res1))
}) }
} }
object Finder { object Finder {
@ -102,7 +124,11 @@ object FindProposal {
labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten) labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten)
} }
private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { private def search[F[_]: Sync](
nt: NerLabel,
exact: Boolean,
ctx: Context[F, ProcessItemArgs]
): F[MetaProposalList] = {
val value = val value =
if (exact) normalizeSearchValue(nt.label) if (exact) normalizeSearchValue(nt.label)
else s"%${normalizeSearchValue(nt.label)}%" else s"%${normalizeSearchValue(nt.label)}%"
@ -110,70 +136,84 @@ object FindProposal {
if (exact) 2 else 5 if (exact) 2 else 5
if (value.length < minLength) { if (value.length < minLength) {
ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty) ctx.logger
} else nt.tag match { .debug(s"Skipping too small value '$value' (original '${nt.label}').")
case NerTag.Organization => .map(_ => MetaProposalList.empty)
ctx.logger.debug(s"Looking for organizations: $value") *> } else
ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)). nt.tag match {
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) case NerTag.Organization =>
ctx.logger.debug(s"Looking for organizations: $value") *>
ctx.store
.transact(ROrganization.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
case NerTag.Person => case NerTag.Person =>
val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)). val s1 = ctx.store
map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) .transact(RPerson.findLike(ctx.args.meta.collective, value, true))
val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)). .map(MetaProposalList.from(MetaProposalType.ConcPerson, nt))
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) val s2 = ctx.store
ctx.logger.debug(s"Looking for persons: $value") *> (for { .transact(RPerson.findLike(ctx.args.meta.collective, value, false))
ml0 <- s1 .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ml1 <- s2 ctx.logger.debug(s"Looking for persons: $value") *> (for {
} yield ml0 |+| ml1) ml0 <- s1
ml1 <- s2
} yield ml0 |+| ml1)
case NerTag.Location => case NerTag.Location =>
ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'."). ctx.logger
map(_ => MetaProposalList.empty) .debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.")
.map(_ => MetaProposalList.empty)
case NerTag.Misc => case NerTag.Misc =>
ctx.logger.debug(s"Looking for equipments: $value") *> ctx.logger.debug(s"Looking for equipments: $value") *>
ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)). ctx.store
map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) .transact(REquipment.findLike(ctx.args.meta.collective, value))
.map(MetaProposalList.from(MetaProposalType.ConcEquip, nt))
case NerTag.Email => case NerTag.Email =>
searchContact(nt, ContactKind.Email, value, ctx) searchContact(nt, ContactKind.Email, value, ctx)
case NerTag.Website => case NerTag.Website =>
if (!exact) { if (!exact) {
val searchString = Domain.domainFromUri(nt.label.toLowerCase). val searchString = Domain
toOption. .domainFromUri(nt.label.toLowerCase)
map(_.toPrimaryDomain.asString). .toOption
map(s => s"%$s%"). .map(_.toPrimaryDomain.asString)
getOrElse(value) .map(s => s"%$s%")
searchContact(nt, ContactKind.Website, searchString, ctx) .getOrElse(value)
} else { searchContact(nt, ContactKind.Website, searchString, ctx)
searchContact(nt, ContactKind.Website, value, ctx) } else {
} searchContact(nt, ContactKind.Website, value, ctx)
}
case NerTag.Date => case NerTag.Date =>
// There is no database search required for this tag // There is no database search required for this tag
MetaProposalList.empty.pure[F] MetaProposalList.empty.pure[F]
} }
} }
private def searchContact[F[_]: Sync]( nt: NerLabel private def searchContact[F[_]: Sync](
, kind: ContactKind nt: NerLabel,
, value: String kind: ContactKind,
, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { value: String,
val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)). ctx: Context[F, ProcessItemArgs]
map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) ): F[MetaProposalList] = {
val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)). val orgs = ctx.store
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) .transact(ROrganization.findLike(ctx.args.meta.collective, kind, value))
val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)). .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt))
map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) val corrP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
val concP = ctx.store
.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true))
.map(MetaProposalList.from(MetaProposalType.CorrPerson, nt))
ctx.logger.debug(s"Looking with $kind: $value") *> ctx.logger.debug(s"Looking with $kind: $value") *>
List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten) List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten)
} }
// The backslash *must* be stripped from search strings. // The backslash *must* be stripped from search strings.
private [this] val invalidSearch = private[this] val invalidSearch =
"…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet "…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet
private def normalizeSearchValue(str: String): String = private def normalizeSearchValue(str: String): String =

View File

@ -4,10 +4,12 @@ import docspell.common.{Ident, NerDateLabel, NerLabel}
import docspell.joex.process.ItemData.AttachmentDates import docspell.joex.process.ItemData.AttachmentDates
import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
case class ItemData( item: RItem case class ItemData(
, attachments: Vector[RAttachment] item: RItem,
, metas: Vector[RAttachmentMeta] attachments: Vector[RAttachment],
, dateLabels: Vector[AttachmentDates]) { metas: Vector[RAttachmentMeta],
dateLabels: Vector[AttachmentDates]
) {
def findMeta(attachId: Ident): Option[RAttachmentMeta] = def findMeta(attachId: Ident): Option[RAttachmentMeta] =
metas.find(_.id == attachId) metas.find(_.id == attachId)
@ -16,7 +18,6 @@ case class ItemData( item: RItem
dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty) dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty)
} }
object ItemData { object ItemData {
case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) { case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) {
@ -24,4 +25,4 @@ object ItemData {
dates.map(dl => dl.label.copy(label = dl.date.toString)) dates.map(dl => dl.label.copy(label = dl.date.toString))
} }
} }

View File

@ -10,14 +10,13 @@ import docspell.text.ocr.{Config => OcrConfig}
object ItemHandler { object ItemHandler {
def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
logWarn("Now cancelling. Deleting potentially created data."). logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ => deleteByFileIds)
flatMap(_ => deleteByFileIds)
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] = def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] =
CreateItem[F]. CreateItem[F]
flatMap(itemStateTask(ItemState.Processing)). .flatMap(itemStateTask(ItemState.Processing))
flatMap(safeProcess[F](cfg)). .flatMap(safeProcess[F](cfg))
map(_ => ()) .map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] = def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx => Task { ctx =>
@ -26,26 +25,25 @@ object ItemHandler {
def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] = def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
for { for {
current <- ctx.store.transact(RJob.getRetries(ctx.jobId)) current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
last = ctx.config.retries == current.getOrElse(0) last = ctx.config.retries == current.getOrElse(0)
} yield last } yield last
def safeProcess[F[_]: Sync: ContextShift](
def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] = cfg: OcrConfig
)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task(isLastRetry[F, ProcessItemArgs] _).flatMap { Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
case true => case true =>
ProcessItem[F](cfg)(data). ProcessItem[F](cfg)(data).attempt.flatMap({
attempt.flatMap({
case Right(d) => case Right(d) =>
Task.pure(d) Task.pure(d)
case Left(ex) => case Left(ex) =>
logWarn[F]("Processing failed on last retry. Creating item but without proposals."). logWarn[F]("Processing failed on last retry. Creating item but without proposals.")
flatMap(_ => itemStateTask(ItemState.Created)(data)). .flatMap(_ => itemStateTask(ItemState.Created)(data))
andThen(_ => Sync[F].raiseError(ex)) .andThen(_ => Sync[F].raiseError(ex))
}) })
case false => case false =>
ProcessItem[F](cfg)(data). ProcessItem[F](cfg)(data).flatMap(itemStateTask(ItemState.Created))
flatMap(itemStateTask(ItemState.Created))
} }
def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =

View File

@ -13,28 +13,40 @@ object LinkProposal {
val proposals = MetaProposalList.flatten(data.metas.map(_.proposals)) val proposals = MetaProposalList.flatten(data.metas.map(_.proposals))
ctx.logger.info(s"Starting linking proposals") *> ctx.logger.info(s"Starting linking proposals") *>
MetaProposalType.all. MetaProposalType.all
traverse(applyValue(data, proposals, ctx)). .traverse(applyValue(data, proposals, ctx))
map(result => ctx.logger.info(s"Results from proposal processing: $result")). .map(result => ctx.logger.info(s"Results from proposal processing: $result"))
map(_ => data) .map(_ => data)
} }
def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = { def applyValue[F[_]: Sync](
data: ItemData,
proposalList: MetaProposalList,
ctx: Context[F, ProcessItemArgs]
)(mpt: MetaProposalType): F[Result] =
proposalList.find(mpt) match { proposalList.find(mpt) match {
case None => case None =>
Result.noneFound(mpt).pure[F] Result.noneFound(mpt).pure[F]
case Some(a) if a.isSingleValue => case Some(a) if a.isSingleValue =>
ctx.logger.info(s"Found one candidate for ${a.proposalType}") *> ctx.logger.info(s"Found one candidate for ${a.proposalType}") *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
map(_ => Result.single(mpt)) Result.single(mpt)
)
case Some(a) => case Some(a) =>
ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *> ctx.logger.info(
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first."
map(_ => Result.multiple(mpt)) ) *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.multiple(mpt)
)
} }
}
def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] = def setItemMeta[F[_]: Sync](
itemId: Ident,
ctx: Context[F, ProcessItemArgs],
mpt: MetaProposalType,
value: Ident
): F[Int] =
mpt match { mpt match {
case MetaProposalType.CorrOrg => case MetaProposalType.CorrOrg =>
ctx.logger.debug(s"Updating item organization with: ${value.id}") *> ctx.logger.debug(s"Updating item organization with: ${value.id}") *>
@ -54,18 +66,17 @@ object LinkProposal {
ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0) ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0)
} }
sealed trait Result { sealed trait Result {
def proposalType: MetaProposalType def proposalType: MetaProposalType
} }
object Result { object Result {
case class NoneFound(proposalType: MetaProposalType) extends Result case class NoneFound(proposalType: MetaProposalType) extends Result
case class SingleResult(proposalType: MetaProposalType) extends Result case class SingleResult(proposalType: MetaProposalType) extends Result
case class MultipleResult(proposalType: MetaProposalType) extends Result case class MultipleResult(proposalType: MetaProposalType) extends Result
def noneFound(proposalType: MetaProposalType): Result = NoneFound(proposalType) def noneFound(proposalType: MetaProposalType): Result = NoneFound(proposalType)
def single(proposalType: MetaProposalType): Result = SingleResult(proposalType) def single(proposalType: MetaProposalType): Result = SingleResult(proposalType)
def multiple(proposalType: MetaProposalType): Result = MultipleResult(proposalType) def multiple(proposalType: MetaProposalType): Result = MultipleResult(proposalType)
} }
} }

View File

@ -7,13 +7,15 @@ import docspell.text.ocr.{Config => OcrConfig}
object ProcessItem { object ProcessItem {
def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync: ContextShift](
TextExtraction(cfg, item). cfg: OcrConfig
flatMap(Task.setProgress(25)). )(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
flatMap(TextAnalysis[F]). TextExtraction(cfg, item)
flatMap(Task.setProgress(50)). .flatMap(Task.setProgress(25))
flatMap(FindProposal[F]). .flatMap(TextAnalysis[F])
flatMap(Task.setProgress(75)). .flatMap(Task.setProgress(50))
flatMap(LinkProposal[F]). .flatMap(FindProposal[F])
flatMap(Task.setProgress(99)) .flatMap(Task.setProgress(75))
.flatMap(LinkProposal[F])
.flatMap(Task.setProgress(99))
} }

View File

@ -8,7 +8,7 @@ import docspell.joex.scheduler.Task
import org.log4s._ import org.log4s._
object TestTasks { object TestTasks {
private [this] val logger = getLogger private[this] val logger = getLogger
def success[F[_]]: Task[F, ProcessItemArgs, Unit] = def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>
@ -17,23 +17,23 @@ object TestTasks {
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>
ctx.logger.info(s"Failing the task run :(").map(_ => ctx.logger
sys.error("Oh, cannot extract gold from this document") .info(s"Failing the task run :(")
) .map(_ => sys.error("Oh, cannot extract gold from this document"))
} }
def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>
logger.fwarn(s"${Thread.currentThread()} From executing long running task") >> logger.fwarn(s"${Thread.currentThread()} From executing long running task") >>
ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >> ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >>
sleep(2400) >> sleep(2400) >>
ctx.logger.debug("doing things") >> ctx.logger.debug("doing things") >>
sleep(2400) >> sleep(2400) >>
ctx.logger.debug("doing more things") >> ctx.logger.debug("doing more things") >>
sleep(2400) >> sleep(2400) >>
ctx.logger.info("doing more things") ctx.logger.info("doing more things")
} }
private def sleep[F[_]:Sync](ms: Long): F[Unit] = private def sleep[F[_]: Sync](ms: Long): F[Unit] =
Sync[F].delay(Thread.sleep(ms)) Sync[F].delay(Thread.sleep(ms))
} }

View File

@ -15,35 +15,42 @@ object TextAnalysis {
def apply[F[_]: Sync](item: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync](item: ItemData): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
for { for {
_ <- ctx.logger.info("Starting text analysis") _ <- ctx.logger.info("Starting text analysis")
s <- Duration.stopTime[F] s <- Duration.stopTime[F]
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))) _ <- t.traverse(m =>
e <- s ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") )
v = t.toVector e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector
} yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2)) } yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2))
} }
def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = def annotateAttachment[F[_]: Sync](
lang: Language
)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] =
for { for {
list0 <- stanfordNer[F](lang, rm) list0 <- stanfordNer[F](lang, rm)
list1 <- contactNer[F](rm) list1 <- contactNer[F](rm)
dates <- dateNer[F](rm, lang) dates <- dateNer[F](rm, lang)
} yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates) } yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates)
def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] =
rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty) Sync[F].delay {
} rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty)
}
def contactNer[F[_]: Sync](rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { def contactNer[F[_]: Sync](rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay {
rm.content.map(Contact.annotate).getOrElse(Vector.empty) rm.content.map(Contact.annotate).getOrElse(Vector.empty)
} }
def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay { def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay {
AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)) AttachmentDates(
rm,
rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)
)
} }
} }

View File

@ -11,10 +11,13 @@ import docspell.text.ocr.{TextExtract, Config => OcrConfig}
object TextExtraction { object TextExtraction {
def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] = def apply[F[_]: Sync: ContextShift](
cfg: OcrConfig,
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
for { for {
_ <- ctx.logger.info("Starting text extraction") _ <- ctx.logger.info("Starting text extraction")
start <- Duration.stopTime[F] start <- Duration.stopTime[F]
txt <- item.attachments.traverse(extractTextToMeta(ctx, cfg, ctx.args.meta.language)) txt <- item.attachments.traverse(extractTextToMeta(ctx, cfg, ctx.args.meta.language))
_ <- ctx.logger.debug("Storing extracted texts") _ <- ctx.logger.debug("Storing extracted texts")
@ -24,22 +27,33 @@ object TextExtraction {
} yield item.copy(metas = txt) } yield item.copy(metas = txt)
} }
def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] = def extractTextToMeta[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: OcrConfig,
lang: Language
)(ra: RAttachment): F[RAttachmentMeta] =
for { for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}") _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}")
dst <- Duration.stopTime[F] dst <- Duration.stopTime[F]
txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra) txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra)
meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty)) meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty))
est <- dst est <- dst
_ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}") _ <- ctx.logger.debug(
s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}"
)
} yield meta } yield meta
def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = { def extractText[F[_]: Sync: ContextShift](
val data = store.bitpeace.get(ra.fileId.id). ocrConfig: OcrConfig,
unNoneTerminate. lang: Language,
through(store.bitpeace.fetchData2(RangeDef.all)) store: Store[F],
blocker: Blocker
)(ra: RAttachment): F[Option[String]] = {
val data = store.bitpeace
.get(ra.fileId.id)
.unNoneTerminate
.through(store.bitpeace.fetchData2(RangeDef.all))
TextExtract.extract(data, blocker, lang.iso3, ocrConfig). TextExtract.extract(data, blocker, lang.iso3, ocrConfig).compile.last
compile.last
} }
} }

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes { object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = { def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> (Root / "version") => case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version Ok(
, BuildInfo.builtAtMillis VersionInfo(
, BuildInfo.builtAtString BuildInfo.version,
, BuildInfo.gitHeadCommit.getOrElse("") BuildInfo.builtAtMillis,
, BuildInfo.gitDescribedVersion.getOrElse(""))) BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
} }
} }
} }

View File

@ -13,7 +13,7 @@ import org.http4s.dsl.Http4sDsl
object JoexRoutes { object JoexRoutes {
def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = { def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case POST -> Root / "notify" => case POST -> Root / "notify" =>
@ -24,14 +24,16 @@ object JoexRoutes {
case GET -> Root / "running" => case GET -> Root / "running" =>
for { for {
jobs <- app.scheduler.getRunning jobs <- app.scheduler.getRunning
jj = jobs.map(mkJob) jj = jobs.map(mkJob)
resp <- Ok(JobList(jj.toList)) resp <- Ok(JobList(jj.toList))
} yield resp } yield resp
case POST -> Root / "shutdownAndExit" => case POST -> Root / "shutdownAndExit" =>
for { for {
_ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown) _ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
resp <- Ok(BasicResult(true, "Shutdown initiated.")) resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp } yield resp
@ -39,20 +41,28 @@ object JoexRoutes {
for { for {
optJob <- app.scheduler.getRunning.map(_.find(_.id == id)) optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
optLog <- optJob.traverse(j => app.findLogs(j.id)) optLog <- optJob.traverse(j => app.findLogs(j.id))
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found"))) resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case POST -> Root / "job" / Ident(id) / "cancel" => case POST -> Root / "job" / Ident(id) / "cancel" =>
for { for {
flag <- app.scheduler.requestCancel(id) flag <- app.scheduler.requestCancel(id)
resp <- Ok(BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found")) resp <- Ok(BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found"))
} yield resp } yield resp
} }
} }
def mkJob(j: RJob): Job = def mkJob(j: RJob): Job =
Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch)) Job(
j.id,
j.subject,
j.submitted,
j.priority,
j.retries,
j.progress,
j.started.getOrElse(Timestamp.Epoch)
)
def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog = def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog =
JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList) JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList)

View File

@ -30,40 +30,45 @@ trait Context[F[_], A] { self =>
} }
object Context { object Context {
private [this] val log = getLogger private[this] val log = getLogger
def create[F[_]: Functor, A]( job: RJob def create[F[_]: Functor, A](
, arg: A job: RJob,
, config: SchedulerConfig arg: A,
, log: Logger[F] config: SchedulerConfig,
, store: Store[F] log: Logger[F],
, blocker: Blocker): Context[F, A] = store: Store[F],
blocker: Blocker
): Context[F, A] =
new ContextImpl(arg, log, store, blocker, config, job.id) new ContextImpl(arg, log, store, blocker, config, job.id)
def apply[F[_]: Concurrent, A]( job: RJob def apply[F[_]: Concurrent, A](
, arg: A job: RJob,
, config: SchedulerConfig arg: A,
, logSink: LogSink[F] config: SchedulerConfig,
, blocker: Blocker logSink: LogSink[F],
, store: Store[F]): F[Context[F, A]] = blocker: Blocker,
store: Store[F]
): F[Context[F, A]] =
for { for {
_ <- log.ftrace("Creating logger for task run") _ <- log.ftrace("Creating logger for task run")
logger <- Logger(job.id, job.info, config.logBufferSize, logSink) logger <- Logger(job.id, job.info, config.logBufferSize, logSink)
_ <- log.ftrace("Logger created, instantiating context") _ <- log.ftrace("Logger created, instantiating context")
ctx = create[F, A](job, arg, config, logger, store, blocker) ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx } yield ctx
private final class ContextImpl[F[_]: Functor, A]( val args: A final private class ContextImpl[F[_]: Functor, A](
, val logger: Logger[F] val args: A,
, val store: Store[F] val logger: Logger[F],
, val blocker: Blocker val store: Store[F],
, val config: SchedulerConfig val blocker: Blocker,
, val jobId: Ident) val config: SchedulerConfig,
extends Context[F,A] { val jobId: Ident
) extends Context[F, A] {
def setProgress(percent: Int): F[Unit] = { def setProgress(percent: Int): F[Unit] = {
val pval = math.min(100, math.max(0, percent)) val pval = math.min(100, math.max(0, percent))
store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) store.transact(RJob.setProgress(jobId, pval)).map(_ => ())
} }
} }
} }

View File

@ -11,14 +11,13 @@ import docspell.common.Priority
*/ */
case class CountingScheme(high: Int, low: Int, counter: Int = 0) { case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
def nextPriority: (CountingScheme, Priority) = { def nextPriority: (CountingScheme, Priority) =
if (counter <= 0) (increment, Priority.High) if (counter <= 0) (increment, Priority.High)
else { else {
val rest = counter % (high + low) val rest = counter % (high + low)
if (rest < high) (increment, Priority.High) if (rest < high) (increment, Priority.High)
else (increment, Priority.Low) else (increment, Priority.Low)
} }
}
def increment: CountingScheme = def increment: CountingScheme =
copy(counter = counter + 1) copy(counter = counter + 1)
@ -32,8 +31,7 @@ object CountingScheme {
def readString(str: String): Either[String, CountingScheme] = def readString(str: String): Either[String, CountingScheme] =
str.split(',') match { str.split(',') match {
case Array(h, l) => case Array(h, l) =>
Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)). Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage)
left.map(_.getMessage)
case _ => case _ =>
Left(s"Invalid counting scheme: $str") Left(s"Invalid counting scheme: $str")
} }

View File

@ -20,13 +20,16 @@ case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Tas
object JobTask { object JobTask {
def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit]) def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])(
(implicit D: Decoder[A]): JobTask[F] = { implicit D: Decoder[A]
): JobTask[F] = {
val convert: String => F[A] = val convert: String => F[A] =
str => str.parseJsonAs[A] match { str =>
case Right(a) => a.pure[F] str.parseJsonAs[A] match {
case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) case Right(a) => a.pure[F]
} case Left(ex) =>
Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex))
}
JobTask(name, task.contramap(convert), onCancel.contramap(convert)) JobTask(name, task.contramap(convert), onCancel.contramap(convert))
} }

View File

@ -4,12 +4,14 @@ import cats.implicits._
import docspell.common._ import docspell.common._
import cats.effect.Sync import cats.effect.Sync
case class LogEvent( jobId: Ident case class LogEvent(
, jobInfo: String jobId: Ident,
, time: Timestamp jobInfo: String,
, level: LogLevel time: Timestamp,
, msg: String level: LogLevel,
, ex: Option[Throwable] = None) { msg: String,
ex: Option[Throwable] = None
) {
def logLine: String = def logLine: String =
s">>> ${time.asString} $level $jobInfo: $msg" s">>> ${time.asString} $level $jobInfo: $msg"
@ -21,5 +23,4 @@ object LogEvent {
def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] = def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] =
Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg))
} }

View File

@ -44,12 +44,22 @@ object LogSink {
LogSink(_.evalMap(e => logInternal(e))) LogSink(_.evalMap(e => logInternal(e)))
def db[F[_]: Sync](store: Store[F]): LogSink[F] = def db[F[_]: Sync](store: Store[F]): LogSink[F] =
LogSink(_.evalMap(ev => for { LogSink(
id <- Ident.randomId[F] _.evalMap(ev =>
joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse("")) for {
_ <- logInternal(ev) id <- Ident.randomId[F]
_ <- store.transact(RJobLog.insert(joblog)) joblog = RJobLog(
} yield ())) id,
ev.jobId,
ev.level,
ev.time,
ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("")
)
_ <- logInternal(ev)
_ <- store.transact(RJobLog.insert(joblog))
} yield ()
)
)
def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = { def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = {
val s: Stream[F, Pipe[F, LogEvent, Unit]] = val s: Stream[F, Pipe[F, LogEvent, Unit]] =

View File

@ -33,17 +33,25 @@ object Logger {
LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1) LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1)
def error(ex: Throwable)(msg: => String): F[Unit] = def error(ex: Throwable)(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1) LogEvent
.create[F](jobId, jobInfo, LogLevel.Error, msg)
.map(le => le.copy(ex = Some(ex)))
.flatMap(q.enqueue1)
def error(msg: => String): F[Unit] = def error(msg: => String): F[Unit] =
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1) LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1)
} }
def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] = def apply[F[_]: Concurrent](
jobId: Ident,
jobInfo: String,
bufferSize: Int,
sink: LogSink[F]
): F[Logger[F]] =
for { for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize) q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q) log = create(jobId, jobInfo, q)
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain) _ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
} yield log } yield log
} }

View File

@ -7,13 +7,14 @@ import docspell.store.Store
import docspell.store.queue.JobQueue import docspell.store.queue.JobQueue
import fs2.concurrent.SignallingRef import fs2.concurrent.SignallingRef
case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( case class SchedulerBuilder[F[_]: ConcurrentEffect: ContextShift](
config: SchedulerConfig config: SchedulerConfig,
, tasks: JobTaskRegistry[F] tasks: JobTaskRegistry[F],
, store: Store[F] store: Store[F],
, blocker: Blocker blocker: Blocker,
, queue: Resource[F, JobQueue[F]] queue: Resource[F, JobQueue[F]],
, logSink: LogSink[F]) { logSink: LogSink[F]
) {
def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] =
copy(config = cfg) copy(config = cfg)
@ -33,7 +34,6 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] =
copy(logSink = sink) copy(logSink = sink)
def serve: Resource[F, Scheduler[F]] = def serve: Resource[F, Scheduler[F]] =
resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch)) resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch))
@ -45,22 +45,25 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift](
perms <- Resource.liftF(Semaphore(config.poolSize.toLong)) perms <- Resource.liftF(Semaphore(config.poolSize.toLong))
} yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms) } yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms)
scheduler.evalTap(_.init). scheduler.evalTap(_.init).map(s => s: Scheduler[F])
map(s => s: Scheduler[F])
} }
} }
object SchedulerBuilder { object SchedulerBuilder {
def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig def apply[F[_]: ConcurrentEffect: ContextShift](
, blocker: Blocker config: SchedulerConfig,
, store: Store[F]): SchedulerBuilder[F] = blocker: Blocker,
new SchedulerBuilder[F](config store: Store[F]
, JobTaskRegistry.empty[F] ): SchedulerBuilder[F] =
, store new SchedulerBuilder[F](
, blocker config,
, JobQueue(store) JobTaskRegistry.empty[F],
, LogSink.db[F](store)) store,
blocker,
JobQueue(store),
LogSink.db[F](store)
)
} }

View File

@ -2,24 +2,26 @@ package docspell.joex.scheduler
import docspell.common._ import docspell.common._
case class SchedulerConfig( name: Ident case class SchedulerConfig(
, poolSize: Int name: Ident,
, countingScheme: CountingScheme poolSize: Int,
, retries: Int countingScheme: CountingScheme,
, retryDelay: Duration retries: Int,
, logBufferSize: Int retryDelay: Duration,
, wakeupPeriod: Duration logBufferSize: Int,
) wakeupPeriod: Duration
)
object SchedulerConfig { object SchedulerConfig {
val default = SchedulerConfig( val default = SchedulerConfig(
name = Ident.unsafe("default-scheduler") name = Ident.unsafe("default-scheduler"),
, poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2)
, countingScheme = CountingScheme(2, 1) ,
, retries = 5 countingScheme = CountingScheme(2, 1),
, retryDelay = Duration.seconds(30) retries = 5,
, logBufferSize = 500 retryDelay = Duration.seconds(30),
, wakeupPeriod = Duration.minutes(10) logBufferSize = 500,
wakeupPeriod = Duration.minutes(10)
) )
} }

View File

@ -14,17 +14,19 @@ import SchedulerImpl._
import docspell.store.Store import docspell.store.Store
import docspell.store.queries.QJob import docspell.store.queries.QJob
final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
, blocker: Blocker val config: SchedulerConfig,
, queue: JobQueue[F] blocker: Blocker,
, tasks: JobTaskRegistry[F] queue: JobQueue[F],
, store: Store[F] tasks: JobTaskRegistry[F],
, logSink: LogSink[F] store: Store[F],
, state: SignallingRef[F, State[F]] logSink: LogSink[F],
, waiter: SignallingRef[F, Boolean] state: SignallingRef[F, State[F]],
, permits: Semaphore[F]) extends Scheduler[F] { waiter: SignallingRef[F, Boolean],
permits: Semaphore[F]
) extends Scheduler[F] {
private [this] val logger = getLogger private[this] val logger = getLogger
/** /**
* On startup, get all jobs in state running from this scheduler * On startup, get all jobs in state running from this scheduler
@ -34,8 +36,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.runningToWaiting(config.name, store) QJob.runningToWaiting(config.name, store)
def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] = def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] =
ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala). ConcurrentEffect[F].start(
evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain) Stream
.awakeEvery[F](config.wakeupPeriod.toScala)
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
.compile
.drain
)
def getRunning: F[Vector[RJob]] = def getRunning: F[Vector[RJob]] =
state.get.flatMap(s => QJob.findAll(s.getRunning, store)) state.get.flatMap(s => QJob.findAll(s.getRunning, store))
@ -43,7 +50,7 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def requestCancel(jobId: Ident): F[Boolean] = def requestCancel(jobId: Ident): F[Boolean] =
state.get.flatMap(_.cancelRequest(jobId) match { state.get.flatMap(_.cancelRequest(jobId) match {
case Some(ct) => ct.map(_ => true) case Some(ct) => ct.map(_ => true)
case None => logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) case None => logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false)
}) })
def notifyChange: F[Unit] = def notifyChange: F[Unit] =
@ -51,59 +58,72 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def shutdown(cancelAll: Boolean): F[Unit] = { def shutdown(cancelAll: Boolean): F[Unit] = {
val doCancel = val doCancel =
state.get. state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ())
flatMap(_.cancelTokens.values.toList.traverse(identity)).
map(_ => ())
val runShutdown = val runShutdown =
state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F])
val wait = Stream.eval(runShutdown). val wait = Stream
evalMap(_ => logger.finfo("Scheduler is shutting down now.")). .eval(runShutdown)
flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))). .evalMap(_ => logger.finfo("Scheduler is shutting down now."))
flatMap(state => { .flatMap(_ =>
Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
)
.flatMap { state =>
if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running.")) if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running."))
else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++ else
Stream.emit(state) Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++
}) Stream.emit(state)
}
(wait.drain ++ Stream.emit(())).compile.lastOrError (wait.drain ++ Stream.emit(())).compile.lastOrError
} }
def start: Stream[F, Nothing] = def start: Stream[F, Nothing] =
logger.sinfo("Starting scheduler") ++ logger.sinfo("Starting scheduler") ++
mainLoop mainLoop
def mainLoop: Stream[F, Nothing] = { def mainLoop: Stream[F, Nothing] = {
val body: F[Boolean] = val body: F[Boolean] =
for { for {
_ <- permits.available.flatMap(a => logger.fdebug(s"Try to acquire permit ($a free)")) _ <- permits.available.flatMap(a => logger.fdebug(s"Try to acquire permit ($a free)"))
_ <- permits.acquire _ <- permits.acquire
_ <- logger.fdebug("New permit acquired") _ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest) down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay) else
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") queue.nextJob(
_ <- rjob.map(execute).getOrElse(permits.release) group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined } yield rjob.isDefined
Stream.eval(state.get.map(_.shutdownRequest)). Stream
evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]). .eval(state.get.map(_.shutdownRequest))
flatMap(if (_) Stream.empty else Stream.eval(body)). .evalTap(
flatMap({ if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
else ().pure[F]
)
.flatMap(if (_) Stream.empty else Stream.eval(body))
.flatMap({
case true => case true =>
mainLoop mainLoop
case false => case false =>
logger.sdebug(s"Waiting for notify") ++ logger.sdebug(s"Waiting for notify") ++
waiter.discrete.take(2).drain ++ waiter.discrete.take(2).drain ++
logger.sdebug(s"Notify signal, going into main loop") ++ logger.sdebug(s"Notify signal, going into main loop") ++
mainLoop mainLoop
}) })
} }
def execute(job: RJob): F[Unit] = { def execute(job: RJob): F[Unit] = {
val task = for { val task = for {
jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}") jobtask <- tasks
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask } yield jobtask
task match { task match {
@ -122,18 +142,25 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
def onFinish(job: RJob, finalState: JobState): F[Unit] = def onFinish(job: RJob, finalState: JobState): F[Unit] =
for { for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)")) _ <- permits.release *> permits.available.flatMap(a =>
_ <- state.modify(_.removeRunning(job)) logger.fdebug(s"Permit released ($a free)")
_ <- QJob.setFinalState(job.id, finalState, store) )
_ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store)
} yield () } yield ()
def onStart(job: RJob): F[Unit] = def onStart(job: RJob): F[Unit] =
QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck
def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = { def wrapTask(
task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)). job: RJob,
mapF(_.attempt.flatMap({ task: Task[F, String, Unit],
ctx: Context[F, String]
): Task[F, String, Unit] =
task
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa))
.mapF(_.attempt.flatMap({
case Right(()) => case Right(()) =>
logger.info(s"Job execution successful: ${job.info}") logger.info(s"Job execution successful: ${job.info}")
ctx.logger.info("Job execution successful") *> ctx.logger.info("Job execution successful") *>
@ -148,16 +175,18 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
QJob.exceedsRetries(job.id, config.retries, store).flatMap { QJob.exceedsRetries(job.id, config.retries, store).flatMap {
case true => case true =>
logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded."). ctx.logger
map(_ => JobState.Failed: JobState) .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.")
.map(_ => JobState.Failed: JobState)
case false => case false =>
logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.")
ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later."). ctx.logger
map(_ => JobState.Stuck: JobState) .error(ex)(s"Job ${job.info} execution failed. Retrying later.")
.map(_ => JobState.Stuck: JobState)
} }
} }
})). }))
mapF(_.attempt.flatMap { .mapF(_.attempt.flatMap {
case Right(jstate) => case Right(jstate) =>
onFinish(job, jstate) onFinish(job, jstate)
case Left(ex) => case Left(ex) =>
@ -165,14 +194,14 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
// we don't know the real outcome here // we don't know the real outcome here
// since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways
onFinish(job, JobState.Stuck) onFinish(job, JobState.Stuck)
}) })
}
def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = { def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = {
val bfa = blocker.blockOn(code) val bfa = blocker.blockOn(code)
logger.fdebug(s"Forking job ${job.info}") *> logger.fdebug(s"Forking job ${job.info}") *>
ConcurrentEffect[F].start(bfa). ConcurrentEffect[F]
map(fiber => .start(bfa)
.map(fiber =>
logger.fdebug(s"Cancelling job ${job.info}") *> logger.fdebug(s"Cancelling job ${job.info}") *>
fiber.cancel *> fiber.cancel *>
onCancel.attempt.map({ onCancel.attempt.map({
@ -184,7 +213,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch
state.modify(_.markCancelled(job)) *> state.modify(_.markCancelled(job)) *>
onFinish(job, JobState.Cancelled) *> onFinish(job, JobState.Cancelled) *>
ctx.logger.warn("Job has been cancelled.") *> ctx.logger.warn("Job has been cancelled.") *>
logger.fdebug(s"Job ${job.info} has been cancelled.")) logger.fdebug(s"Job ${job.info} has been cancelled.")
)
} }
} }
@ -193,10 +223,12 @@ object SchedulerImpl {
def emptyState[F[_]]: State[F] = def emptyState[F[_]]: State[F] =
State(Map.empty, Set.empty, Map.empty, false) State(Map.empty, Set.empty, Map.empty, false)
case class State[F[_]]( counters: Map[Ident, CountingScheme] case class State[F[_]](
, cancelled: Set[Ident] counters: Map[Ident, CountingScheme],
, cancelTokens: Map[Ident, CancelToken[F]] cancelled: Set[Ident],
, shutdownRequest: Boolean) { cancelTokens: Map[Ident, CancelToken[F]],
shutdownRequest: Boolean
) {
def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = {
val (cs, prio) = counters.getOrElse(group, initial).nextPriority val (cs, prio) = counters.getOrElse(group, initial).nextPriority

View File

@ -24,11 +24,11 @@ trait Task[F[_], A, B] {
def mapF[C](f: F[B] => F[C]): Task[F, A, C] = def mapF[C](f: F[B] => F[C]): Task[F, A, C] =
Task(Task.toKleisli(this).mapF(f)) Task(Task.toKleisli(this).mapF(f))
def attempt(implicit F: ApplicativeError[F,Throwable]): Task[F, A, Either[Throwable, B]] = def attempt(implicit F: ApplicativeError[F, Throwable]): Task[F, A, Either[Throwable, B]] =
mapF(_.attempt) mapF(_.attempt)
def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { ctxc: Context[F, C] =>
ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) f(ctxc.args).flatMap(a => run(ctxc.map(_ => a)))
} }
} }
@ -46,7 +46,6 @@ object Task {
def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] =
c => k.run(c) c => k.run(c)
def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] =
Kleisli(t.run) Kleisli(t.run)

View File

@ -6,8 +6,8 @@ import minitest.SimpleTestSuite
object CountingSchemeSpec extends SimpleTestSuite { object CountingSchemeSpec extends SimpleTestSuite {
test("counting") { test("counting") {
val cs = CountingScheme(2,1) val cs = CountingScheme(2, 1)
val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2) val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2)
val expect = List(Priority.High, Priority.High, Priority.Low) val expect = List(Priority.High, Priority.High, Priority.Low)
assertEquals(list, expect ++ expect) assertEquals(list, expect ++ expect)
} }

View File

@ -7,27 +7,37 @@ import docspell.backend.{Config => BackendConfig}
import docspell.common._ import docspell.common._
import scodec.bits.ByteVector import scodec.bits.ByteVector
case class Config(appName: String case class Config(
, appId: Ident appName: String,
, baseUrl: LenientUri appId: Ident,
, bind: Config.Bind baseUrl: LenientUri,
, backend: BackendConfig bind: Config.Bind,
, auth: Login.Config backend: BackendConfig,
auth: Login.Config
) )
object Config { object Config {
val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") val postgres =
val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev")
val h2 = JdbcConfig(
LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"),
"sa",
""
)
val default: Config = val default: Config =
Config("Docspell" Config(
, Ident.unsafe("restserver1") "Docspell",
, LenientUri.unsafe("http://localhost:7880") Ident.unsafe("restserver1"),
, Config.Bind("localhost", 7880) LenientUri.unsafe("http://localhost:7880"),
, BackendConfig(postgres Config.Bind("localhost", 7880),
, SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)) BackendConfig(
, BackendConfig.Files(512 * 1024, List(MimeType.pdf))) postgres,
, Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))) SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)),
BackendConfig.Files(512 * 1024, List(MimeType.pdf))
),
Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))
)
case class Bind(address: String, port: Int) case class Bind(address: String, port: Int)
} }

View File

@ -12,7 +12,7 @@ object ConfigFile {
ConfigSource.default.at("docspell.server").loadOrThrow[Config] ConfigSource.default.at("docspell.server").loadOrThrow[Config]
object Implicits { object Implicits {
implicit val signupModeReader: ConfigReader[SignupConfig.Mode] = implicit val signupModeReader: ConfigReader[SignupConfig.Mode] =
ConfigReader[String].emap(reason(SignupConfig.Mode.fromString)) ConfigReader[String].emap(reason(SignupConfig.Mode.fromString))
} }
} }

View File

@ -13,12 +13,14 @@ import org.log4s._
object Main extends IOApp { object Main extends IOApp {
private[this] val logger = getLogger private[this] val logger = getLogger
val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool( val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(
ThreadFactories.ofName("docspell-restserver-blocking"))) Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-restserver-blocking"))
)
val blocker = Blocker.liftExecutionContext(blockingEc) val blocker = Blocker.liftExecutionContext(blockingEc)
val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5, val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(
ThreadFactories.ofName("docspell-dbconnect"))) Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-dbconnect"))
)
def run(args: List[String]) = { def run(args: List[String]) = {
args match { args match {
@ -41,12 +43,15 @@ object Main extends IOApp {
} }
val cfg = ConfigFile.loadConfig val cfg = ConfigFile.loadConfig
val banner = Banner("REST Server" val banner = Banner(
, BuildInfo.version "REST Server",
, BuildInfo.gitHeadCommit BuildInfo.version,
, cfg.backend.jdbc.url BuildInfo.gitHeadCommit,
, Option(System.getProperty("config.file")) cfg.backend.jdbc.url,
, cfg.appId, cfg.baseUrl) Option(System.getProperty("config.file")),
cfg.appId,
cfg.baseUrl
)
logger.info(s"\n${banner.render("***>")}") logger.info(s"\n${banner.render("***>")}")
RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success) RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success)
} }

View File

@ -7,7 +7,8 @@ import docspell.common.NodeType
import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext
final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F])
extends RestApp[F] {
def init: F[Unit] = def init: F[Unit] =
backend.node.register(config.appId, NodeType.Restserver, config.baseUrl) backend.node.register(config.appId, NodeType.Restserver, config.baseUrl)
@ -18,11 +19,16 @@ final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[
object RestAppImpl { object RestAppImpl {
def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] = def create[F[_]: ConcurrentEffect: ContextShift](
cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
): Resource[F, RestApp[F]] =
for { for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend) app = new RestAppImpl[F](cfg, backend)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown) appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR } yield appR
} }

View File

@ -15,54 +15,64 @@ import scala.concurrent.ExecutionContext
object RestServer { object RestServer {
def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker) def stream[F[_]: ConcurrentEffect](
(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = { cfg: Config,
connectEC: ExecutionContext,
httpClientEc: ExecutionContext,
blocker: Blocker
)(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = {
val app = for { val app = for {
restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker) restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker)
httpApp = Router( httpApp = Router(
"/api/info" -> routes.InfoRoutes(), "/api/info" -> routes.InfoRoutes(),
"/api/v1/open/" -> openRoutes(cfg, restApp), "/api/v1/open/" -> openRoutes(cfg, restApp),
"/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
token => securedRoutes(cfg, restApp, token) securedRoutes(cfg, restApp, token)
}, },
"/app/assets" -> WebjarRoutes.appRoutes[F](blocker), "/app/assets" -> WebjarRoutes.appRoutes[F](blocker),
"/app" -> TemplateRoutes[F](blocker, cfg) "/app" -> TemplateRoutes[F](blocker, cfg)
).orNotFound ).orNotFound
finalHttpApp = Logger.httpApp(logHeaders = false, logBody = false)(httpApp) finalHttpApp = Logger.httpApp(logHeaders = false, logBody = false)(httpApp)
} yield finalHttpApp } yield finalHttpApp
Stream.resource(app).flatMap(httpApp => Stream
BlazeServerBuilder[F]. .resource(app)
bindHttp(cfg.bind.port, cfg.bind.address). .flatMap(httpApp =>
withHttpApp(httpApp). BlazeServerBuilder[F]
withoutBanner. .bindHttp(cfg.bind.port, cfg.bind.address)
serve) .withHttpApp(httpApp)
.withoutBanner
.serve
)
}.drain }.drain
def securedRoutes[F[_]: Effect](
def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] = cfg: Config,
restApp: RestApp[F],
token: AuthToken
): HttpRoutes[F] =
Router( Router(
"auth" -> LoginRoutes.session(restApp.backend.login, cfg), "auth" -> LoginRoutes.session(restApp.backend.login, cfg),
"tag" -> TagRoutes(restApp.backend, token), "tag" -> TagRoutes(restApp.backend, token),
"equipment" -> EquipmentRoutes(restApp.backend, token), "equipment" -> EquipmentRoutes(restApp.backend, token),
"organization" -> OrganizationRoutes(restApp.backend, token), "organization" -> OrganizationRoutes(restApp.backend, token),
"person" -> PersonRoutes(restApp.backend, token), "person" -> PersonRoutes(restApp.backend, token),
"source" -> SourceRoutes(restApp.backend, token), "source" -> SourceRoutes(restApp.backend, token),
"user" -> UserRoutes(restApp.backend, token), "user" -> UserRoutes(restApp.backend, token),
"collective" -> CollectiveRoutes(restApp.backend, token), "collective" -> CollectiveRoutes(restApp.backend, token),
"queue" -> JobQueueRoutes(restApp.backend, token), "queue" -> JobQueueRoutes(restApp.backend, token),
"item" -> ItemRoutes(restApp.backend, token), "item" -> ItemRoutes(restApp.backend, token),
"attachment" -> AttachmentRoutes(restApp.backend, token), "attachment" -> AttachmentRoutes(restApp.backend, token),
"upload" -> UploadRoutes.secured(restApp.backend, cfg, token) "upload" -> UploadRoutes.secured(restApp.backend, cfg, token)
) )
def openRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] = def openRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] =
Router( Router(
"auth" -> LoginRoutes.login(restApp.backend.login, cfg), "auth" -> LoginRoutes.login(restApp.backend.login, cfg),
"signup" -> RegisterRoutes(restApp.backend, cfg), "signup" -> RegisterRoutes(restApp.backend, cfg),
"upload" -> UploadRoutes.open(restApp.backend, cfg) "upload" -> UploadRoutes.open(restApp.backend, cfg)
) )

View File

@ -8,13 +8,20 @@ import docspell.restserver.Config
case class CookieData(auth: AuthToken) { case class CookieData(auth: AuthToken) {
def accountId: AccountId = auth.account def accountId: AccountId = auth.account
def asString: String = auth.asString def asString: String = auth.asString
def asCookie(cfg: Config): ResponseCookie = { def asCookie(cfg: Config): ResponseCookie = {
val domain = cfg.baseUrl.host val domain = cfg.baseUrl.host
val sec = cfg.baseUrl.scheme.exists(_.endsWith("s")) val sec = cfg.baseUrl.scheme.exists(_.endsWith("s"))
val path = cfg.baseUrl.path/"api"/"v1"/"sec" val path = cfg.baseUrl.path / "api" / "v1" / "sec"
ResponseCookie(CookieData.cookieName, asString, domain = domain, path = Some(path.asString), httpOnly = true, secure = sec) ResponseCookie(
CookieData.cookieName,
asString,
domain = domain,
path = Some(path.asString),
httpOnly = true,
secure = sec
)
} }
} }
object CookieData { object CookieData {
@ -22,18 +29,21 @@ object CookieData {
val headerName = "X-Docspell-Auth" val headerName = "X-Docspell-Auth"
def authenticator[F[_]](r: Request[F]): Either[String, String] = def authenticator[F[_]](r: Request[F]): Either[String, String] =
fromCookie(r) orElse fromHeader(r) fromCookie(r).orElse(fromHeader(r))
def fromCookie[F[_]](req: Request[F]): Either[String, String] = { def fromCookie[F[_]](req: Request[F]): Either[String, String] =
for { for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie") cookie <- header.values.toList
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
} yield cookie.content } yield cookie.content
}
def fromHeader[F[_]](req: Request[F]): Either[String, String] = { def fromHeader[F[_]](req: Request[F]): Either[String, String] =
req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator") req.headers
} .get(CaseInsensitiveString(headerName))
.map(_.value)
.toRight("Couldn't find an authenticator")
def deleteCookie(cfg: Config): ResponseCookie = def deleteCookie(cfg: Config): ResponseCookie =
ResponseCookie( ResponseCookie(

View File

@ -24,31 +24,37 @@ trait Conversions {
// insights // insights
def mkItemInsights(d: InsightData): ItemInsights = def mkItemInsights(d: InsightData): ItemInsights =
ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))) ItemInsights(
d.incoming,
d.outgoing,
d.bytes,
TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))
)
// attachment meta // attachment meta
def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta =
AttachmentMeta(rm.content.getOrElse("") AttachmentMeta(
, rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)) rm.content.getOrElse(""),
, mkItemProposals(rm.proposals)) rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)),
mkItemProposals(rm.proposals)
)
// item proposal // item proposal
def mkItemProposals(ml: MetaProposalList): ItemProposals = { def mkItemProposals(ml: MetaProposalList): ItemProposals = {
def get(mpt: MetaProposalType) = def get(mpt: MetaProposalType) =
ml.find(mpt). ml.find(mpt).map(mp => mp.values.toList.map(_.ref).map(mkIdName)).getOrElse(Nil)
map(mp => mp.values.toList.map(_.ref).map(mkIdName)).
getOrElse(Nil)
def getDates(mpt: MetaProposalType): List[Timestamp] = def getDates(mpt: MetaProposalType): List[Timestamp] =
ml.find(mpt). ml.find(mpt)
map(mp => mp.values.toList. .map(mp =>
map(cand => cand.ref.id.id). mp.values.toList
flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption). .map(cand => cand.ref.id.id)
map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))). .flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption)
map(zdt => Timestamp(zdt.toInstant))). .map(_.atTime(12, 0).atZone(ZoneId.of("GMT")))
getOrElse(Nil). .map(zdt => Timestamp(zdt.toInstant))
distinct. )
take(5) .getOrElse(Nil)
.distinct
.take(5)
ItemProposals( ItemProposals(
corrOrg = get(MetaProposalType.CorrOrg), corrOrg = get(MetaProposalType.CorrOrg),
@ -62,23 +68,25 @@ trait Conversions {
// item detail // item detail
def mkItemDetail(data: OItem.ItemData): ItemDetail = def mkItemDetail(data: OItem.ItemData): ItemDetail =
ItemDetail(data.item.id ItemDetail(
, data.item.direction data.item.id,
, data.item.name data.item.direction,
, data.item.source data.item.name,
, data.item.state data.item.source,
, data.item.created data.item.state,
, data.item.updated data.item.created,
, data.item.itemDate data.item.updated,
, data.corrOrg.map(o => IdName(o.oid, o.name)) data.item.itemDate,
, data.corrPerson.map(p => IdName(p.pid, p.name)) data.corrOrg.map(o => IdName(o.oid, o.name)),
, data.concPerson.map(p => IdName(p.pid, p.name)) data.corrPerson.map(p => IdName(p.pid, p.name)),
, data.concEquip.map(e => IdName(e.eid, e.name)) data.concPerson.map(p => IdName(p.pid, p.name)),
, data.inReplyTo.map(mkIdName) data.concEquip.map(e => IdName(e.eid, e.name)),
, data.item.dueDate data.inReplyTo.map(mkIdName),
, data.item.notes data.item.dueDate,
, data.attachments.map((mkAttachment _).tupled).toList data.item.notes,
, data.tags.map(mkTag).toList) data.attachments.map((mkAttachment _).tupled).toList,
data.tags.map(mkTag).toList
)
def mkAttachment(ra: RAttachment, m: FileMeta): Attachment = def mkAttachment(ra: RAttachment, m: FileMeta): Attachment =
Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString)) Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString))
@ -86,20 +94,21 @@ trait Conversions {
// item list // item list
def mkQuery(m: ItemSearch, coll: Ident): OItem.Query = def mkQuery(m: ItemSearch, coll: Ident): OItem.Query =
OItem.Query(coll OItem.Query(
, m.name coll,
, if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed) m.name,
, m.direction if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed),
, m.corrPerson m.direction,
, m.corrOrg m.corrPerson,
, m.concPerson m.corrOrg,
, m.concEquip m.concPerson,
, m.tagsInclude.map(Ident.unsafe) m.concEquip,
, m.tagsExclude.map(Ident.unsafe) m.tagsInclude.map(Ident.unsafe),
, m.dateFrom m.tagsExclude.map(Ident.unsafe),
, m.dateUntil m.dateFrom,
, m.dueDateFrom m.dateUntil,
, m.dueDateUntil m.dueDateFrom,
m.dueDateUntil
) )
def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = { def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = {
@ -113,8 +122,20 @@ trait Conversions {
} }
def mkItemLight(i: OItem.ListItem): ItemLight = def mkItemLight(i: OItem.ListItem): ItemLight =
ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName), ItemLight(
i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount) i.id,
i.name,
i.state,
i.date,
i.dueDate,
i.source,
i.direction.name.some,
i.corrOrg.map(mkIdName),
i.corrPerson.map(mkIdName),
i.concPerson.map(mkIdName),
i.concEquip.map(mkIdName),
i.fileCount
)
// job // job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
@ -128,46 +149,57 @@ trait Conversions {
val t2 = f(j2).getOrElse(Timestamp.Epoch) val t2 = f(j2).getOrElse(Timestamp.Epoch)
t1.value.isBefore(t2.value) t1.value.isBefore(t2.value)
} }
JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started)) JobQueueState(
, state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)) state.running.map(mkJobDetail).toList.sortWith(asc(_.started)),
, state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))) state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)),
state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))
)
} }
def mkJobDetail(jd: OJob.JobDetail): JobDetail = def mkJobDetail(jd: OJob.JobDetail): JobDetail =
JobDetail(jd.job.id JobDetail(
, jd.job.subject jd.job.id,
, jd.job.submitted jd.job.subject,
, jd.job.priority jd.job.submitted,
, jd.job.state jd.job.priority,
, jd.job.retries jd.job.state,
, jd.logs.map(mkJobLog).toList jd.job.retries,
, jd.job.progress jd.logs.map(mkJobLog).toList,
, jd.job.worker jd.job.progress,
, jd.job.started jd.job.worker,
, jd.job.finished) jd.job.started,
jd.job.finished
)
def mkJobLog(jl: RJobLog): JobLogEvent = def mkJobLog(jl: RJobLog): JobLogEvent =
JobLogEvent(jl.created, jl.level, jl.message) JobLogEvent(jl.created, jl.level, jl.message)
// upload // upload
def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = { def readMultipart[F[_]: Effect](
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = { mp: Multipart[F],
body.through(fs2.text.utf8Decode). logger: Logger,
parseJsonAs[ItemUploadMeta]. prio: Priority,
map(_.fold(ex => { validFileTypes: Seq[MimeType]
): F[UploadData[F]] = {
def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] =
body
.through(fs2.text.utf8Decode)
.parseJsonAs[ItemUploadMeta]
.map(_.fold(ex => {
logger.error(ex)("Reading upload metadata failed.") logger.error(ex)("Reading upload metadata failed.")
throw ex throw ex
}, identity)) }, identity))
}
val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")). val meta: F[(Boolean, UploadMeta)] = mp.parts
map(p => parseMeta(p.body)). .find(_.name.exists(_.equalsIgnoreCase("meta")))
map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))). .map(p => parseMeta(p.body))
getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) .map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes))))
.getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
val files = mp.parts. val files = mp.parts
filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))). .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)) .map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
)
for { for {
metaData <- meta metaData <- meta
_ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData")) _ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData"))
@ -178,8 +210,14 @@ trait Conversions {
// organization and person // organization and person
def mkOrg(v: OOrganization.OrgAndContacts): Organization = { def mkOrg(v: OOrganization.OrgAndContacts): Organization = {
val ro = v.org val ro = v.org
Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), Organization(
v.contacts.map(mkContact).toList, ro.notes, ro.created) ro.oid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.created
)
} }
def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = { def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = {
@ -189,7 +227,17 @@ trait Conversions {
now <- Timestamp.current[F] now <- Timestamp.current[F]
oid <- Ident.randomId[F] oid <- Ident.randomId[F]
cont <- contacts(oid) cont <- contacts(oid)
org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now) org = ROrganization(
oid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
now
)
} yield OOrganization.OrgAndContacts(org, cont) } yield OOrganization.OrgAndContacts(org, cont)
} }
@ -197,15 +245,32 @@ trait Conversions {
def contacts(oid: Ident) = def contacts(oid: Ident) =
v.contacts.traverse(c => newContact(c, oid.some, None)) v.contacts.traverse(c => newContact(c, oid.some, None))
for { for {
cont <- contacts(v.id) cont <- contacts(v.id)
org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created) org = ROrganization(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.created
)
} yield OOrganization.OrgAndContacts(org, cont) } yield OOrganization.OrgAndContacts(org, cont)
} }
def mkPerson(v: OOrganization.PersonAndContacts): Person = { def mkPerson(v: OOrganization.PersonAndContacts): Person = {
val ro = v.person val ro = v.person
Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), Person(
v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created) ro.pid,
ro.name,
Address(ro.street, ro.zip, ro.city, ro.country),
v.contacts.map(mkContact).toList,
ro.notes,
ro.concerning,
ro.created
)
} }
def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = { def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = {
@ -215,7 +280,18 @@ trait Conversions {
now <- Timestamp.current[F] now <- Timestamp.current[F]
pid <- Ident.randomId[F] pid <- Ident.randomId[F]
cont <- contacts(pid) cont <- contacts(pid)
org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now) org = RPerson(
pid,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
now
)
} yield OOrganization.PersonAndContacts(org, cont) } yield OOrganization.PersonAndContacts(org, cont)
} }
@ -223,8 +299,19 @@ trait Conversions {
def contacts(pid: Ident) = def contacts(pid: Ident) =
v.contacts.traverse(c => newContact(c, None, pid.some)) v.contacts.traverse(c => newContact(c, None, pid.some))
for { for {
cont <- contacts(v.id) cont <- contacts(v.id)
org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created) org = RPerson(
v.id,
cid,
v.name,
v.address.street,
v.address.zip,
v.address.city,
v.address.country,
v.notes,
v.concerning,
v.created
)
} yield OOrganization.PersonAndContacts(org, cont) } yield OOrganization.PersonAndContacts(org, cont)
} }
@ -233,7 +320,8 @@ trait Conversions {
Contact(rc.contactId, rc.value, rc.kind) Contact(rc.contactId, rc.value, rc.kind)
def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] = def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] =
timeId.map { case (id, now) => timeId.map {
case (id, now) =>
RContact(id, c.value, c.kind, pid, oid, now) RContact(id, c.value, c.kind, pid, oid, now)
} }
@ -242,12 +330,33 @@ trait Conversions {
User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created) User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created)
def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] =
timeId.map { case (id, now) => timeId.map {
RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, 0, None, now) case (id, now) =>
RUser(
id,
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
0,
None,
now
)
} }
def changeUser(u: User, cid: Ident): RUser = def changeUser(u: User, cid: Ident): RUser =
RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created) RUser(
Ident.unsafe(""),
u.login,
cid,
u.password.getOrElse(Password.empty),
u.state,
u.email,
u.loginCount,
u.lastLogin,
u.created
)
// tags // tags
@ -255,34 +364,36 @@ trait Conversions {
Tag(rt.tagId, rt.name, rt.category, rt.created) Tag(rt.tagId, rt.name, rt.category, rt.created)
def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] =
timeId.map { case (id, now) => timeId.map {
RTag(id, cid, t.name, t.category, now) case (id, now) =>
RTag(id, cid, t.name, t.category, now)
} }
def changeTag(t: Tag, cid: Ident): RTag = def changeTag(t: Tag, cid: Ident): RTag =
RTag(t.id, cid, t.name, t.category, t.created) RTag(t.id, cid, t.name, t.category, t.created)
// sources // sources
def mkSource(s: RSource): Source = def mkSource(s: RSource): Source =
Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map({ case (id, now) => timeId.map({
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) case (id, now) =>
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
}) })
def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource = def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
RSource(s.id, coll, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) RSource(s.id, coll, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
// equipment // equipment
def mkEquipment(re: REquipment): Equipment = def mkEquipment(re: REquipment): Equipment =
Equipment(re.eid, re.name, re.created) Equipment(re.eid, re.name, re.created)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map({ case (id, now) => timeId.map({
REquipment(id, cid, e.name, now) case (id, now) =>
REquipment(id, cid, e.name, now)
}) })
def changeEquipment(e: Equipment, cid: Ident): REquipment = def changeEquipment(e: Equipment, cid: Ident): REquipment =
@ -298,26 +409,28 @@ trait Conversions {
def basicResult(cr: JobCancelResult): BasicResult = def basicResult(cr: JobCancelResult): BasicResult =
cr match { cr match {
case JobCancelResult.JobNotFound => BasicResult(false, "Job not found") case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor") case JobCancelResult.CancelRequested =>
BasicResult(true, "Cancel was requested at the job executor")
case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.") case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.")
} }
def basicResult(ar: AddResult, successMsg: String): BasicResult = ar match { def basicResult(ar: AddResult, successMsg: String): BasicResult = ar match {
case AddResult.Success => BasicResult(true, successMsg) case AddResult.Success => BasicResult(true, successMsg)
case AddResult.EntityExists(msg) => BasicResult(false, msg) case AddResult.EntityExists(msg) => BasicResult(false, msg)
case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}") case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}")
} }
def basicResult(ur: OUpload.UploadResult): BasicResult = ur match { def basicResult(ur: OUpload.UploadResult): BasicResult = ur match {
case UploadResult.Success => BasicResult(true, "Files submitted.") case UploadResult.Success => BasicResult(true, "Files submitted.")
case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.") case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.")
case UploadResult.NoSource => BasicResult(false, "The source id is not valid.") case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
} }
def basicResult(cr: PassChangeResult): BasicResult = cr match { def basicResult(cr: PassChangeResult): BasicResult = cr match {
case PassChangeResult.Success => BasicResult(true, "Password changed.") case PassChangeResult.Success => BasicResult(true, "Password changed.")
case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.") case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.")
case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.") case PassChangeResult.PasswordMismatch =>
BasicResult(false, "The current password is incorrect.")
case PassChangeResult.UserNotFound => BasicResult(false, "User not found.") case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
} }

View File

@ -8,28 +8,26 @@ import org.http4s.dsl.Http4sDsl
trait ResponseGenerator[F[_]] { trait ResponseGenerator[F[_]] {
self: Http4sDsl[F] => self: Http4sDsl[F] =>
implicit final class EitherResponses[A, B](e: Either[A, B]) {
implicit final class EitherResponses[A,B](e: Either[A, B]) { def toResponse(headers: Header*)(
def toResponse(headers: Header*) implicit F: Applicative[F],
(implicit F: Applicative[F] w0: EntityEncoder[F, A],
, w0: EntityEncoder[F, A] w1: EntityEncoder[F, B]
, w1: EntityEncoder[F, B]): F[Response[F]] = ): F[Response[F]] =
e.fold( e.fold(
a => UnprocessableEntity(a), a => UnprocessableEntity(a),
b => Ok(b) b => Ok(b)
).map(_.withHeaders(headers: _*)) )
.map(_.withHeaders(headers: _*))
} }
implicit final class OptionResponse[A](o: Option[A]) { implicit final class OptionResponse[A](o: Option[A]) {
def toResponse(headers: Header*) def toResponse(
(implicit F: Applicative[F] headers: Header*
, w0: EntityEncoder[F, A]): F[Response[F]] = )(implicit F: Applicative[F], w0: EntityEncoder[F, A]): F[Response[F]] =
o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*)) o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*))
} }
} }
object ResponseGenerator { object ResponseGenerator {}
}

View File

@ -18,17 +18,18 @@ import org.http4s.headers.ETag.EntityTag
object AttachmentRoutes { object AttachmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = { def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = {
val mt = MediaType.unsafeParse(data.meta.mimetype.asString) val mt = MediaType.unsafeParse(data.meta.mimetype.asString)
val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length) val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length)
val eTag: Header = ETag(data.meta.checksum) val eTag: Header = ETag(data.meta.checksum)
val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse(""))) val disp: Header =
Ok(data.data.take(data.meta.length)). `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse("")))
map(r => r.withContentType(`Content-Type`(mt)). Ok(data.data.take(data.meta.length)).map(r =>
withHeaders(cntLen, eTag, disp)) r.withContentType(`Content-Type`(mt)).withHeaders(cntLen, eTag, disp)
)
} }
HttpRoutes.of { HttpRoutes.of {
@ -37,21 +38,24 @@ object AttachmentRoutes {
fileData <- backend.item.findAttachment(id, user.account.collective) fileData <- backend.item.findAttachment(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags) inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData, inm) matches = matchETag(fileData, inm)
resp <- if (matches) NotModified() resp <- if (matches) NotModified()
else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found"))) else
fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case GET -> Root / Ident(id) / "meta" => case GET -> Root / Ident(id) / "meta" =>
for { for {
rm <- backend.item.findAttachmentMeta(id, user.account.collective) rm <- backend.item.findAttachmentMeta(id, user.account.collective)
md = rm.map(Conversions.mkAttachmentMeta) md = rm.map(Conversions.mkAttachmentMeta)
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found."))) resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp } yield resp
} }
} }
private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]] private def matchETag[F[_]](
, noneMatch: Option[NonEmptyList[EntityTag]]): Boolean = fileData: Option[OItem.AttachmentData[F]],
noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean =
(fileData, noneMatch) match { (fileData, noneMatch) match {
case (Some(fd), Some(nm)) => case (Some(fd), Some(nm)) =>
fd.meta.checksum == nm.head.tag fd.meta.checksum == nm.head.tag

View File

@ -12,14 +12,17 @@ import org.http4s.server._
object Authenticate { object Authenticate {
def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] = def authenticateRequest[F[_]: Effect](
auth: String => F[Login.Result]
)(req: Request[F]): F[Login.Result] =
CookieData.authenticator(req) match { CookieData.authenticator(req) match {
case Right(str) => auth(str) case Right(str) => auth(str)
case Left(_) => Login.Result.invalidAuth.pure[F] case Left(_) => Login.Result.invalidAuth.pure[F]
} }
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(
def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = { pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {} val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._ import dsl._
@ -34,7 +37,9 @@ object Authenticate {
middleware(AuthedRoutes.of(pf)) middleware(AuthedRoutes.of(pf))
} }
def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = { def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(
f: AuthToken => HttpRoutes[F]
): HttpRoutes[F] = {
val dsl: Http4sDsl[F] = new Http4sDsl[F] {} val dsl: Http4sDsl[F] = new Http4sDsl[F] {}
import dsl._ import dsl._
@ -49,6 +54,8 @@ object Authenticate {
middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req))) middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req)))
} }
private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] = private def getUser[F[_]: Effect](
auth: String => F[Login.Result]
): Kleisli[F, Request[F], Either[String, AuthToken]] =
Kleisli(r => authenticateRequest(auth)(r).map(_.toEither)) Kleisli(r => authenticateRequest(auth)(r).map(_.toEither))
} }

View File

@ -25,25 +25,25 @@ object CollectiveRoutes {
resp <- Ok(Conversions.mkItemInsights(ins)) resp <- Ok(Conversions.mkItemInsights(ins))
} yield resp } yield resp
case req@POST -> Root / "settings" => case req @ POST -> Root / "settings" =>
for { for {
settings <- req.as[CollectiveSettings] settings <- req.as[CollectiveSettings]
res <- backend.collective.updateLanguage(user.account.collective, settings.language) res <- backend.collective.updateLanguage(user.account.collective, settings.language)
resp <- Ok(Conversions.basicResult(res, "Language updated.")) resp <- Ok(Conversions.basicResult(res, "Language updated."))
} yield resp } yield resp
case GET -> Root / "settings" => case GET -> Root / "settings" =>
for { for {
collDb <- backend.collective.find(user.account.collective) collDb <- backend.collective.find(user.account.collective)
sett = collDb.map(c => CollectiveSettings(c.language)) sett = collDb.map(c => CollectiveSettings(c.language))
resp <- sett.toResponse() resp <- sett.toResponse()
} yield resp } yield resp
case GET -> Root => case GET -> Root =>
for { for {
collDb <- backend.collective.find(user.account.collective) collDb <- backend.collective.find(user.account.collective)
coll = collDb.map(c => Collective(c.id, c.state, c.created)) coll = collDb.map(c => Collective(c.id, c.state, c.created))
resp <- coll.toResponse() resp <- coll.toResponse()
} yield resp } yield resp
} }
} }

View File

@ -15,7 +15,7 @@ import org.http4s.dsl.Http4sDsl
object EquipmentRoutes { object EquipmentRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
@ -36,12 +36,12 @@ object EquipmentRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Equipment] data <- req.as[Equipment]
equip = changeEquipment(data, user.account.collective) equip = changeEquipment(data, user.account.collective)
res <- backend.equipment.update(equip) res <- backend.equipment.update(equip)
resp <- Ok(basicResult(res, "Equipment updated.")) resp <- Ok(basicResult(res, "Equipment updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
del <- backend.equipment.delete(id, user.account.collective) del <- backend.equipment.delete(id, user.account.collective)
resp <- Ok(basicResult(del, "Equipment deleted.")) resp <- Ok(basicResult(del, "Equipment deleted."))

View File

@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl
object InfoRoutes { object InfoRoutes {
def apply[F[_]: Sync](): HttpRoutes[F] = { def apply[F[_]: Sync](): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> (Root / "version") => case GET -> (Root / "version") =>
Ok(VersionInfo(BuildInfo.version Ok(
, BuildInfo.builtAtMillis VersionInfo(
, BuildInfo.builtAtString BuildInfo.version,
, BuildInfo.gitHeadCommit.getOrElse("") BuildInfo.builtAtMillis,
, BuildInfo.gitDescribedVersion.getOrElse(""))) BuildInfo.builtAtString,
BuildInfo.gitHeadCommit.getOrElse(""),
BuildInfo.gitDescribedVersion.getOrElse("")
)
)
} }
} }
} }

View File

@ -18,24 +18,24 @@ object ItemRoutes {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "search" => case req @ POST -> Root / "search" =>
for { for {
mask <- req.as[ItemSearch] mask <- req.as[ItemSearch]
_ <- logger.ftrace(s"Got search mask: $mask") _ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective) query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query") _ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItems(query, 100) items <- backend.item.findItems(query, 100)
resp <- Ok(Conversions.mkItemList(items)) resp <- Ok(Conversions.mkItemList(items))
} yield resp } yield resp
case GET -> Root / Ident(id) => case GET -> Root / Ident(id) =>
for { for {
item <- backend.item.findItem(id, user.account.collective) item <- backend.item.findItem(id, user.account.collective)
result = item.map(Conversions.mkItemDetail) result = item.map(Conversions.mkItemDetail)
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found."))) resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp } yield resp
@ -51,89 +51,89 @@ object ItemRoutes {
resp <- Ok(Conversions.basicResult(res, "Item back to created.")) resp <- Ok(Conversions.basicResult(res, "Item back to created."))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "tags" => case req @ POST -> Root / Ident(id) / "tags" =>
for { for {
tags <- req.as[ReferenceList].map(_.items) tags <- req.as[ReferenceList].map(_.items)
res <- backend.item.setTags(id, tags.map(_.id), user.account.collective) res <- backend.item.setTags(id, tags.map(_.id), user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Tags updated")) resp <- Ok(Conversions.basicResult(res, "Tags updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "direction" => case req @ POST -> Root / Ident(id) / "direction" =>
for { for {
dir <- req.as[DirectionValue] dir <- req.as[DirectionValue]
res <- backend.item.setDirection(id, dir.direction, user.account.collective) res <- backend.item.setDirection(id, dir.direction, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Direction updated")) resp <- Ok(Conversions.basicResult(res, "Direction updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "corrOrg" => case req @ POST -> Root / Ident(id) / "corrOrg" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setCorrOrg(id, idref.id, user.account.collective) res <- backend.item.setCorrOrg(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated")) resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "corrPerson" => case req @ POST -> Root / Ident(id) / "corrPerson" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setCorrPerson(id, idref.id, user.account.collective) res <- backend.item.setCorrPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Correspondent person updated")) resp <- Ok(Conversions.basicResult(res, "Correspondent person updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "concPerson" => case req @ POST -> Root / Ident(id) / "concPerson" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setConcPerson(id, idref.id, user.account.collective) res <- backend.item.setConcPerson(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned person updated")) resp <- Ok(Conversions.basicResult(res, "Concerned person updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "concEquipment" => case req @ POST -> Root / Ident(id) / "concEquipment" =>
for { for {
idref <- req.as[OptionalId] idref <- req.as[OptionalId]
res <- backend.item.setConcEquip(id, idref.id, user.account.collective) res <- backend.item.setConcEquip(id, idref.id, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp
case req@POST -> Root / Ident(id) / "notes" =>
for {
text <- req.as[OptionalText]
res <- backend.item.setNotes(id, text.text, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "name" => case req @ POST -> Root / Ident(id) / "notes" =>
for { for {
text <- req.as[OptionalText] text <- req.as[OptionalText]
res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective) res <- backend.item.setNotes(id, text.text, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "duedate" => case req @ POST -> Root / Ident(id) / "name" =>
for { for {
date <- req.as[OptionalDate] text <- req.as[OptionalText]
_ <- logger.fdebug(s"Setting item due date to ${date.date}") res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective)
res <- backend.item.setItemDueDate(id, date.date, user.account.collective) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated"))
resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
} yield resp } yield resp
case req@POST -> Root / Ident(id) / "date" => case req @ POST -> Root / Ident(id) / "duedate" =>
for { for {
date <- req.as[OptionalDate] date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item date to ${date.date}") _ <- logger.fdebug(s"Setting item due date to ${date.date}")
res <- backend.item.setItemDate(id, date.date, user.account.collective) res <- backend.item.setItemDueDate(id, date.date, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Item date updated")) resp <- Ok(Conversions.basicResult(res, "Item due date updated"))
} yield resp
case req @ POST -> Root / Ident(id) / "date" =>
for {
date <- req.as[OptionalDate]
_ <- logger.fdebug(s"Setting item date to ${date.date}")
res <- backend.item.setItemDate(id, date.date, user.account.collective)
resp <- Ok(Conversions.basicResult(res, "Item date updated"))
} yield resp } yield resp
case GET -> Root / Ident(id) / "proposals" => case GET -> Root / Ident(id) / "proposals" =>
for { for {
ml <- backend.item.getProposals(id, user.account.collective) ml <- backend.item.getProposals(id, user.account.collective)
ip = Conversions.mkItemProposals(ml) ip = Conversions.mkItemProposals(ml)
resp <- Ok(ip) resp <- Ok(ip)
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
n <- backend.item.delete(id, user.account.collective) n <- backend.item.delete(id, user.account.collective)
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.") res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
resp <- Ok(res) resp <- Ok(res)
} yield resp } yield resp
} }

View File

@ -19,15 +19,15 @@ object JobQueueRoutes {
HttpRoutes.of { HttpRoutes.of {
case GET -> Root / "state" => case GET -> Root / "state" =>
for { for {
js <- backend.job.queueState(user.account.collective, 200) js <- backend.job.queueState(user.account.collective, 200)
res = Conversions.mkJobQueueState(js) res = Conversions.mkJobQueueState(js)
resp <- Ok(res) resp <- Ok(res)
} yield resp } yield resp
case POST -> Root / Ident(id) / "cancel" => case POST -> Root / Ident(id) / "cancel" =>
for { for {
result <- backend.job.cancelJob(id, user.account.collective) result <- backend.job.cancelJob(id, user.account.collective)
resp <- Ok(Conversions.basicResult(result)) resp <- Ok(Conversions.basicResult(result))
} yield resp } yield resp
} }
} }

View File

@ -18,10 +18,10 @@ object LoginRoutes {
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case req@POST -> Root / "login" => case req @ POST -> Root / "login" =>
for { for {
up <- req.as[UserPass] up <- req.as[UserPass]
res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password)) res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password))
resp <- makeResponse(dsl, cfg, res, up.account) resp <- makeResponse(dsl, cfg, res, up.account)
} yield resp } yield resp
} }
@ -33,22 +33,36 @@ object LoginRoutes {
HttpRoutes.of[F] { HttpRoutes.of[F] {
case req @ POST -> Root / "session" => case req @ POST -> Root / "session" =>
Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req). Authenticate
flatMap(res => makeResponse(dsl, cfg, res, "")) .authenticateRequest(S.loginSession(cfg.auth))(req)
.flatMap(res => makeResponse(dsl, cfg, res, ""))
case POST -> Root / "logout" => case POST -> Root / "logout" =>
Ok().map(_.addCookie(CookieData.deleteCookie(cfg))) Ok().map(_.addCookie(CookieData.deleteCookie(cfg)))
} }
} }
def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = { def makeResponse[F[_]: Effect](
dsl: Http4sDsl[F],
cfg: Config,
res: Login.Result,
account: String
): F[Response[F]] = {
import dsl._ import dsl._
res match { res match {
case Login.Result.Ok(token) => case Login.Result.Ok(token) =>
for { for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply) cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)). resp <- Ok(
map(_.addCookie(cd.asCookie(cfg))) AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
} yield resp } yield resp
case _ => case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L)) Ok(AuthResult("", account, false, "Login failed.", None, 0L))

View File

@ -16,15 +16,15 @@ import org.http4s.dsl.Http4sDsl
object OrganizationRoutes { object OrganizationRoutes {
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
case GET -> Root :? FullQueryParamMatcher(full) => case GET -> Root :? FullQueryParamMatcher(full) =>
if (full.getOrElse(false)) { if (full.getOrElse(false)) {
for { for {
data <- backend.organization.findAllOrg(user.account) data <- backend.organization.findAllOrg(user.account)
resp <- Ok(OrganizationList(data.map(mkOrg).toList)) resp <- Ok(OrganizationList(data.map(mkOrg).toList))
} yield resp } yield resp
} else { } else {
for { for {
@ -38,7 +38,7 @@ object OrganizationRoutes {
data <- req.as[Organization] data <- req.as[Organization]
newOrg <- newOrg(data, user.account.collective) newOrg <- newOrg(data, user.account.collective)
added <- backend.organization.addOrg(newOrg) added <- backend.organization.addOrg(newOrg)
resp <- Ok(basicResult(added, "New organization saved.")) resp <- Ok(basicResult(added, "New organization saved."))
} yield resp } yield resp
case req @ PUT -> Root => case req @ PUT -> Root =>
@ -49,10 +49,10 @@ object OrganizationRoutes {
resp <- Ok(basicResult(update, "Organization updated.")) resp <- Ok(basicResult(update, "Organization updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
delOrg <- backend.organization.deleteOrg(id, user.account.collective) delOrg <- backend.organization.deleteOrg(id, user.account.collective)
resp <- Ok(basicResult(delOrg, "Organization deleted.")) resp <- Ok(basicResult(delOrg, "Organization deleted."))
} yield resp } yield resp
} }
} }

View File

@ -6,9 +6,10 @@ import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher
object ParamDecoder { object ParamDecoder {
implicit val booleanDecoder: QueryParamDecoder[Boolean] = implicit val booleanDecoder: QueryParamDecoder[Boolean] =
QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean") QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_.equalsIgnoreCase("true")))(
"Boolean"
)
object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full") object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full")
} }

View File

@ -19,15 +19,15 @@ object PersonRoutes {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = {
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
case GET -> Root :? FullQueryParamMatcher(full) => case GET -> Root :? FullQueryParamMatcher(full) =>
if (full.getOrElse(false)) { if (full.getOrElse(false)) {
for { for {
data <- backend.organization.findAllPerson(user.account) data <- backend.organization.findAllPerson(user.account)
resp <- Ok(PersonList(data.map(mkPerson).toList)) resp <- Ok(PersonList(data.map(mkPerson).toList))
} yield resp } yield resp
} else { } else {
for { for {
@ -41,7 +41,7 @@ object PersonRoutes {
data <- req.as[Person] data <- req.as[Person]
newPer <- newPerson(data, user.account.collective) newPer <- newPerson(data, user.account.collective)
added <- backend.organization.addPerson(newPer) added <- backend.organization.addPerson(newPer)
resp <- Ok(basicResult(added, "New person saved.")) resp <- Ok(basicResult(added, "New person saved."))
} yield resp } yield resp
case req @ PUT -> Root => case req @ PUT -> Root =>
@ -52,11 +52,11 @@ object PersonRoutes {
resp <- Ok(basicResult(update, "Person updated.")) resp <- Ok(basicResult(update, "Person updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
_ <- logger.fdebug(s"Deleting person ${id.id}") _ <- logger.fdebug(s"Deleting person ${id.id}")
delOrg <- backend.organization.deletePerson(id, user.account.collective) delOrg <- backend.organization.deletePerson(id, user.account.collective)
resp <- Ok(basicResult(delOrg, "Person deleted.")) resp <- Ok(basicResult(delOrg, "Person deleted."))
} yield resp } yield resp
} }
} }

View File

@ -24,16 +24,16 @@ object RegisterRoutes {
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "register" => case req @ POST -> Root / "register" =>
for { for {
data <- req.as[Registration] data <- req.as[Registration]
res <- backend.signup.register(cfg.backend.signup)(convert(data)) res <- backend.signup.register(cfg.backend.signup)(convert(data))
resp <- Ok(convert(res)) resp <- Ok(convert(res))
} yield resp } yield resp
case req@ POST -> Root / "newinvite" => case req @ POST -> Root / "newinvite" =>
for { for {
data <- req.as[GenInvite] data <- req.as[GenInvite]
res <- backend.signup.newInvite(cfg.backend.signup)(data.password) res <- backend.signup.newInvite(cfg.backend.signup)(data.password)
resp <- Ok(convert(res)) resp <- Ok(convert(res))
} yield resp } yield resp
} }
} }
@ -47,7 +47,6 @@ object RegisterRoutes {
InviteResult(false, "Password is invalid.", None) InviteResult(false, "Password is invalid.", None)
} }
def convert(r: SignupResult): BasicResult = r match { def convert(r: SignupResult): BasicResult = r match {
case SignupResult.CollectiveExists => case SignupResult.CollectiveExists =>
BasicResult(false, "A collective with this name already exists.") BasicResult(false, "A collective with this name already exists.")
@ -62,7 +61,6 @@ object RegisterRoutes {
BasicResult(true, "Signup successful") BasicResult(true, "Signup successful")
} }
def convert(r: Registration): RegisterData = def convert(r: Registration): RegisterData =
RegisterData(r.collectiveName, r.login, r.password, r.invite) RegisterData(r.collectiveName, r.login, r.password, r.invite)
} }

View File

@ -22,8 +22,8 @@ object SourceRoutes {
HttpRoutes.of { HttpRoutes.of {
case GET -> Root => case GET -> Root =>
for { for {
all <- backend.source.findAll(user.account) all <- backend.source.findAll(user.account)
res <- Ok(SourceList(all.map(mkSource).toList)) res <- Ok(SourceList(all.map(mkSource).toList))
} yield res } yield res
case req @ POST -> Root => case req @ POST -> Root =>
@ -37,12 +37,12 @@ object SourceRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Source] data <- req.as[Source]
src = changeSource(data, user.account.collective) src = changeSource(data, user.account.collective)
updated <- backend.source.update(src) updated <- backend.source.update(src)
resp <- Ok(basicResult(updated, "Source updated.")) resp <- Ok(basicResult(updated, "Source updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
del <- backend.source.delete(id, user.account.collective) del <- backend.source.delete(id, user.account.collective)
resp <- Ok(basicResult(del, "Source deleted.")) resp <- Ok(basicResult(del, "Source deleted."))

View File

@ -28,21 +28,21 @@ object TagRoutes {
case req @ POST -> Root => case req @ POST -> Root =>
for { for {
data <- req.as[Tag] data <- req.as[Tag]
tag <- newTag(data, user.account.collective) tag <- newTag(data, user.account.collective)
res <- backend.tag.add(tag) res <- backend.tag.add(tag)
resp <- Ok(basicResult(res, "Tag successfully created.")) resp <- Ok(basicResult(res, "Tag successfully created."))
} yield resp } yield resp
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Tag] data <- req.as[Tag]
tag = changeTag(data, user.account.collective) tag = changeTag(data, user.account.collective)
res <- backend.tag.update(tag) res <- backend.tag.update(tag)
resp <- Ok(basicResult(res, "Tag successfully updated.")) resp <- Ok(basicResult(res, "Tag successfully updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
del <- backend.tag.delete(id, user.account.collective) del <- backend.tag.delete(id, user.account.collective)
resp <- Ok(basicResult(del, "Tag successfully deleted.")) resp <- Ok(basicResult(del, "Tag successfully deleted."))

View File

@ -26,9 +26,14 @@ object UploadRoutes {
case req @ POST -> Root / "item" => case req @ POST -> Root / "item" =>
for { for {
multipart <- req.as[Multipart[F]] multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes) updata <- readMultipart(
result <- backend.upload.submit(updata, user.account) multipart,
res <- Ok(basicResult(result)) logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result))
} yield res } yield res
} }
@ -39,12 +44,12 @@ object UploadRoutes {
import dsl._ import dsl._
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "item" / Ident(id)=> case req @ POST -> Root / "item" / Ident(id) =>
for { for {
multipart <- req.as[Multipart[F]] multipart <- req.as[Multipart[F]]
updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes) updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes)
result <- backend.upload.submit(updata, id) result <- backend.upload.submit(updata, id)
res <- Ok(basicResult(result)) res <- Ok(basicResult(result))
} yield res } yield res
} }
} }

View File

@ -22,15 +22,19 @@ object UserRoutes {
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "changePassword" => case req @ POST -> Root / "changePassword" =>
for { for {
data <- req.as[PasswordChange] data <- req.as[PasswordChange]
res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword) res <- backend.collective.changePassword(
resp <- Ok(basicResult(res)) user.account,
data.currentPassword,
data.newPassword
)
resp <- Ok(basicResult(res))
} yield resp } yield resp
case GET -> Root => case GET -> Root =>
for { for {
all <- backend.collective.listUser(user.account.collective) all <- backend.collective.listUser(user.account.collective)
res <- Ok(UserList(all.map(mkUser).toList)) res <- Ok(UserList(all.map(mkUser).toList))
} yield res } yield res
case req @ POST -> Root => case req @ POST -> Root =>
@ -51,7 +55,7 @@ object UserRoutes {
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
ar <- backend.collective.deleteUser(id, user.account.collective) ar <- backend.collective.deleteUser(id, user.account.collective)
resp <- Ok(basicResult(ar, "User deleted.")) resp <- Ok(basicResult(ar, "User deleted."))
} yield resp } yield resp
} }

View File

@ -8,14 +8,21 @@ import docspell.backend.signup.{Config => SignupConfig}
import yamusca.imports._ import yamusca.imports._
import yamusca.implicits._ import yamusca.implicits._
case class Flags( appName: String case class Flags(
, baseUrl: LenientUri appName: String,
, signupMode: SignupConfig.Mode baseUrl: LenientUri,
, docspellAssetPath: String) signupMode: SignupConfig.Mode,
docspellAssetPath: String
)
object Flags { object Flags {
def apply(cfg: Config): Flags = def apply(cfg: Config): Flags =
Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode, s"assets/docspell-webapp/${BuildInfo.version}") Flags(
cfg.appName,
cfg.baseUrl,
cfg.backend.signup.mode,
s"assets/docspell-webapp/${BuildInfo.version}"
)
implicit val jsonEncoder: Encoder[Flags] = implicit val jsonEncoder: Encoder[Flags] =
deriveEncoder[Flags] deriveEncoder[Flags]

View File

@ -21,90 +21,100 @@ object TemplateRoutes {
val `text/html` = new MediaType("text", "html") val `text/html` = new MediaType("text", "html")
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit C: ContextShift[F]): HttpRoutes[F] = { def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(
implicit C: ContextShift[F]
): HttpRoutes[F] = {
val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker))) val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker)))
val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker))) val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker)))
val dsl = new Http4sDsl[F]{} val dsl = new Http4sDsl[F] {}
import dsl._ import dsl._
HttpRoutes.of[F] { HttpRoutes.of[F] {
case GET -> Root / "index.html" => case GET -> Root / "index.html" =>
for { for {
templ <- indexTemplate templ <- indexTemplate
resp <- Ok(IndexData(cfg).render(templ), `Content-Type`(`text/html`)) resp <- Ok(IndexData(cfg).render(templ), `Content-Type`(`text/html`))
} yield resp } yield resp
case GET -> Root / "doc" => case GET -> Root / "doc" =>
for { for {
templ <- docTemplate templ <- docTemplate
resp <- Ok(DocData().render(templ), `Content-Type`(`text/html`)) resp <- Ok(DocData().render(templ), `Content-Type`(`text/html`))
} yield resp } yield resp
} }
} }
def loadResource[F[_]: Sync](name: String): F[URL] = { def loadResource[F[_]: Sync](name: String): F[URL] =
Option(getClass.getResource(name)) match { Option(getClass.getResource(name)) match {
case None => case None =>
Sync[F].raiseError(new Exception("Unknown resource: "+ name)) Sync[F].raiseError(new Exception("Unknown resource: " + name))
case Some(r) => case Some(r) =>
r.pure[F] r.pure[F]
} }
}
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] = def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] =
Stream.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())). Stream
flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)). .bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
through(text.utf8Decode). .flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false))
compile.fold("")(_ + _) .through(text.utf8Decode)
.compile
.fold("")(_ + _)
def parseTemplate[F[_]: Sync](str: String): F[Template] = def parseTemplate[F[_]: Sync](str: String): F[Template] =
Sync[F].delay { Sync[F].delay {
mustache.parse(str) match { mustache.parse(str) match {
case Right(t) => t case Right(t) => t
case Left((_, err)) => sys.error(err) case Left((_, err)) => sys.error(err)
} }
} }
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[Template] = { def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)). implicit C: ContextShift[F]
map(t => { ): F[Template] =
logger.info(s"Compiled template $url") loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
t logger.info(s"Compiled template $url")
}) t
} }
case class DocData(swaggerRoot: String, openapiSpec: String) case class DocData(swaggerRoot: String, openapiSpec: String)
object DocData { object DocData {
def apply(): DocData = def apply(): DocData =
DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml") DocData(
"/app/assets" + Webjars.swaggerui,
s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml"
)
implicit def yamuscaValueConverter: ValueConverter[DocData] = implicit def yamuscaValueConverter: ValueConverter[DocData] =
ValueConverter.deriveConverter[DocData] ValueConverter.deriveConverter[DocData]
} }
case class IndexData(flags: Flags case class IndexData(
, cssUrls: Seq[String] flags: Flags,
, jsUrls: Seq[String] cssUrls: Seq[String],
, faviconBase: String jsUrls: Seq[String],
, appExtraJs: String faviconBase: String,
, flagsJson: String) appExtraJs: String,
flagsJson: String
)
object IndexData { object IndexData {
def apply(cfg: Config): IndexData = def apply(cfg: Config): IndexData =
IndexData(Flags(cfg) IndexData(
, Seq( Flags(cfg),
Seq(
"/app/assets" + Webjars.semanticui + "/semantic.min.css", "/app/assets" + Webjars.semanticui + "/semantic.min.css",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css" s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css"
) ),
, Seq( Seq(
"/app/assets" + Webjars.jquery + "/jquery.min.js", "/app/assets" + Webjars.jquery + "/jquery.min.js",
"/app/assets" + Webjars.semanticui + "/semantic.min.js", "/app/assets" + Webjars.semanticui + "/semantic.min.js",
s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js" s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js"
) ),
, s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon" s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon",
, s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js" s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js",
, Flags(cfg).asJson.spaces2 ) Flags(cfg).asJson.spaces2
)
implicit def yamuscaValueConverter: ValueConverter[IndexData] = implicit def yamuscaValueConverter: ValueConverter[IndexData] =
ValueConverter.deriveConverter[IndexData] ValueConverter.deriveConverter[IndexData]
@ -116,10 +126,10 @@ object TemplateRoutes {
Option(ref.get) match { Option(ref.get) match {
case Some(a) => a.pure[F] case Some(a) => a.pure[F]
case None => case None =>
fa.map(a => { fa.map { a =>
ref.set(a) ref.set(a)
a a
}) }
} }
} }
} }

View File

@ -9,7 +9,7 @@ import org.http4s.server.staticcontent.WebjarService.{WebjarAsset, Config => Web
object WebjarRoutes { object WebjarRoutes {
def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = { def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] =
webjarService( webjarService(
WebjarConfig( WebjarConfig(
filter = assetFilter, filter = assetFilter,
@ -17,10 +17,23 @@ object WebjarRoutes {
cacheStrategy = NoopCacheStrategy[F] cacheStrategy = NoopCacheStrategy[F]
) )
) )
}
def assetFilter(asset: WebjarAsset): Boolean = def assetFilter(asset: WebjarAsset): Boolean =
List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml"). List(
exists(e => asset.asset.endsWith(e)) ".js",
".css",
".html",
".json",
".jpg",
".png",
".eot",
".woff",
".woff2",
".svg",
".otf",
".ttf",
".yml",
".xml"
).exists(e => asset.asset.endsWith(e))
} }

View File

@ -18,14 +18,14 @@ object AddResult {
e.fold(Failure, n => if (n > 0) Success else Failure(new Exception("No rows updated"))) e.fold(Failure, n => if (n > 0) Success else Failure(new Exception("No rows updated")))
case object Success extends AddResult { case object Success extends AddResult {
def toEither = Right(()) def toEither = Right(())
val isSuccess = true val isSuccess = true
def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A =
fa(this) fa(this)
} }
case class EntityExists(msg: String) extends AddResult { case class EntityExists(msg: String) extends AddResult {
def toEither = Left(new Exception(msg)) def toEither = Left(new Exception(msg))
val isSuccess = false val isSuccess = false
def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A =
fb(this) fb(this)
@ -35,7 +35,7 @@ object AddResult {
} }
case class Failure(ex: Throwable) extends AddResult { case class Failure(ex: Throwable) extends AddResult {
def toEither = Left(ex) def toEither = Left(ex)
val isSuccess = false val isSuccess = false
def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A =
fc(this) fc(this)

View File

@ -2,10 +2,7 @@ package docspell.store
import docspell.common.LenientUri import docspell.common.LenientUri
case class JdbcConfig(url: LenientUri case class JdbcConfig(url: LenientUri, user: String, password: String) {
, user: String
, password: String
) {
val dbmsName: Option[String] = val dbmsName: Option[String] =
JdbcConfig.extractDbmsName(url) JdbcConfig.extractDbmsName(url)

View File

@ -22,21 +22,25 @@ trait Store[F[_]] {
object Store { object Store {
def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig def create[F[_]: Effect: ContextShift](
, connectEC: ExecutionContext jdbc: JdbcConfig,
, blocker: Blocker): Resource[F, Store[F]] = { connectEC: ExecutionContext,
blocker: Blocker
): Resource[F, Store[F]] = {
val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass val hxa = HikariTransactor.newHikariTransactor[F](
, jdbc.url.asString jdbc.driverClass,
, jdbc.user jdbc.url.asString,
, jdbc.password jdbc.user,
, connectEC jdbc.password,
, blocker) connectEC,
blocker
)
for { for {
xa <- hxa xa <- hxa
st = new StoreImpl[F](jdbc, xa) st = new StoreImpl[F](jdbc, xa)
_ <- Resource.liftF(st.migrate) _ <- Resource.liftF(st.migrate)
} yield st } yield st
} }
} }

View File

@ -24,7 +24,7 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def is[A: Put](ov: Option[A]): Fragment = ov match { def is[A: Put](ov: Option[A]): Fragment = ov match {
case Some(v) => f ++ fr" = $v" case Some(v) => f ++ fr" = $v"
case None => fr"is null" case None => fr"is null"
} }
def is(c: Column): Fragment = def is(c: Column): Fragment =
@ -42,7 +42,7 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def isOrDiscard[A: Put](value: Option[A]): Fragment = def isOrDiscard[A: Put](value: Option[A]): Fragment =
value match { value match {
case Some(v) => is(v) case Some(v) => is(v)
case None => Fragment.empty case None => Fragment.empty
} }
def isOneOf[A: Put](values: Seq[A]): Fragment = { def isOneOf[A: Put](values: Seq[A]): Fragment = {

View File

@ -21,7 +21,9 @@ trait DoobieMeta {
}) })
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces) Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
e.apply(a).noSpaces
)
implicit val metaCollectiveState: Meta[CollectiveState] = implicit val metaCollectiveState: Meta[CollectiveState] =
Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString) Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString)
@ -45,7 +47,9 @@ trait DoobieMeta {
Meta[String].imap(JobState.unsafe)(_.name) Meta[String].imap(JobState.unsafe)(_.name)
implicit val metaDirection: Meta[Direction] = implicit val metaDirection: Meta[Direction] =
Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d)) Meta[Boolean].imap(flag =>
if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction
)(d => Direction.isIncoming(d))
implicit val metaPriority: Meta[Priority] = implicit val metaPriority: Meta[Priority] =
Meta[Int].imap(Priority.fromInt)(Priority.toInt) Meta[Int].imap(Priority.fromInt)(Priority.toInt)

View File

@ -19,7 +19,9 @@ trait DoobieSyntax {
commas(fa :: fas.toList) commas(fa :: fas.toList)
def and(fs: Seq[Fragment]): Fragment = def and(fs: Seq[Fragment]): Fragment =
Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") Fragment.const(" (") ++ fs
.filter(f => !isEmpty(f))
.reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ")
def and(f0: Fragment, fs: Fragment*): Fragment = def and(f0: Fragment, fs: Fragment*): Fragment =
and(f0 :: fs.toList) and(f0 :: fs.toList)
@ -48,8 +50,9 @@ trait DoobieSyntax {
def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment = def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment =
Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++
commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")")) commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(
vals.map(f => sql"(" ++ f ++ sql")")
)
def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment =
selectSimple(commas(cols.map(_.f)), table, where) selectSimple(commas(cols.map(_.f)), table, where)
@ -62,7 +65,6 @@ trait DoobieSyntax {
Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++ Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++
Fragment.const(") FROM ") ++ table ++ this.where(where) Fragment.const(") FROM ") ++ table ++ this.where(where)
// def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment = // def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment =
// selectSimple(cols.map(_.prefix("a")) // selectSimple(cols.map(_.prefix("a"))
// , table ++ fr"a," ++ RCollective.table ++ fr"b" // , table ++ fr"a," ++ RCollective.table ++ fr"b"
@ -70,11 +72,12 @@ trait DoobieSyntax {
// else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b"))) // else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b")))
def selectCount(col: Column, table: Fragment, where: Fragment): Fragment = def selectCount(col: Column, table: Fragment, where: Fragment): Fragment =
Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where) Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(
where
)
def deleteFrom(table: Fragment, where: Fragment): Fragment = { def deleteFrom(table: Fragment, where: Fragment): Fragment =
fr"DELETE FROM" ++ table ++ this.where(where) fr"DELETE FROM" ++ table ++ this.where(where)
}
def withCTE(ps: (String, Fragment)*): Fragment = { def withCTE(ps: (String, Fragment)*): Fragment = {
val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")") val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")")

View File

@ -1,5 +1,3 @@
package docspell.store.impl package docspell.store.impl
object Implicits extends DoobieMeta with DoobieSyntax
object Implicits extends DoobieMeta
with DoobieSyntax

View File

@ -10,7 +10,8 @@ import doobie._
import doobie.implicits._ import doobie.implicits._
final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] { final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] {
val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id)) val bitpeaceCfg =
BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id))
def migrate: F[Int] = def migrate: F[Int] =
FlywayMigrate.run[F](jdbc) FlywayMigrate.run[F](jdbc)
@ -24,14 +25,14 @@ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends
def bitpeace: Bitpeace[F] = def bitpeace: Bitpeace[F] =
Bitpeace(bitpeaceCfg, xa) Bitpeace(bitpeaceCfg, xa)
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = { def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] =
for { for {
save <- transact(insert).attempt save <- transact(insert).attempt
exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b))) exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b)))
} yield exist.swap match { } yield exist.swap match {
case Right(_) => AddResult.Success case Right(_) => AddResult.Success
case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.") case Left((_, true)) =>
AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((ex, _)) => AddResult.Failure(ex) case Left((ex, _)) => AddResult.Failure(ex)
} }
}
} }

View File

@ -20,11 +20,12 @@ object FlywayMigrate {
} }
logger.info(s"Using migration locations: $locations") logger.info(s"Using migration locations: $locations")
val fw = Flyway.configure(). val fw = Flyway
cleanDisabled(true). .configure()
dataSource(jdbc.url.asString, jdbc.user, jdbc.password). .cleanDisabled(true)
locations(locations: _*). .dataSource(jdbc.url.asString, jdbc.user, jdbc.password)
load() .locations(locations: _*)
.load()
fw.repair() fw.repair()
fw.migrate() fw.migrate()

View File

@ -18,7 +18,7 @@ trait ONode[F[_]] {
object ONode { object ONode {
private[this] val logger = getLogger private[this] val logger = getLogger
def apply[F[_] : Effect](store: Store[F]): Resource[F, ONode[F]] = def apply[F[_]: Effect](store: Store[F]): Resource[F, ONode[F]] =
Resource.pure(new ONode[F] { Resource.pure(new ONode[F] {
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =

View File

@ -12,58 +12,64 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
object QAttachment { object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = { def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for { for {
raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)) raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll))
n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId))) n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId)))
f <- Stream.emit(raOpt). f <- Stream
unNoneTerminate. .emit(raOpt)
map(_.fileId.id). .unNoneTerminate
flatMap(store.bitpeace.delete). .map(_.fileId.id)
compile.last .flatMap(store.bitpeace.delete)
.compile
.last
} yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0) } yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0)
}
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = { def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for { for {
n <- store.transact(RAttachment.delete(ra.id)) n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emit(ra.fileId.id). f <- Stream.emit(ra.fileId.id).flatMap(store.bitpeace.delete).compile.last
flatMap(store.bitpeace.delete).
compile.last
} yield n + f.map(_ => 1).getOrElse(0) } yield n + f.map(_ => 1).getOrElse(0)
}
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = { def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
for { for {
ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll)) ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll))
ns <- ras.traverse(deleteAttachment[F](store)) ns <- ras.traverse(deleteAttachment[F](store))
} yield ns.sum } yield ns.sum
}
def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = { def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = {
val AC = RAttachment.Columns val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns val MC = RAttachmentMeta.Columns
val IC = RItem.Columns val IC = RItem.Columns
val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ val q = fr"SELECT" ++ MC.proposals
.prefix("m")
.f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++
fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++
fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll)) fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll))
for { for {
ml <- q.query[MetaProposalList].to[Vector] ml <- q.query[MetaProposalList].to[Vector]
} yield MetaProposalList.flatten(ml) } yield MetaProposalList.flatten(ml)
} }
def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = { def getAttachmentMeta(
attachId: Ident,
collective: Ident
): ConnectionIO[Option[RAttachmentMeta]] = {
val AC = RAttachment.Columns val AC = RAttachment.Columns
val MC = RAttachmentMeta.Columns val MC = RAttachmentMeta.Columns
val IC = RItem.Columns val IC = RItem.Columns
val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ .prefix("i")
fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective) .is(AC.itemId.prefix("a")) ++
fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id
.prefix("a")
.is(MC.id.prefix("m")) ++
fr"WHERE" ++ and(AC.id.prefix("a").is(attachId), IC.cid.prefix("i").is(collective))
q.query[RAttachmentMeta].option q.query[RAttachmentMeta].option
} }

View File

@ -8,27 +8,35 @@ import docspell.store.records.{RAttachment, RItem, RTag, RTagItem}
object QCollective { object QCollective {
case class InsightData( incoming: Int case class InsightData(incoming: Int, outgoing: Int, bytes: Long, tags: Map[String, Int])
, outgoing: Int
, bytes: Long
, tags: Map[String, Int])
def getInsights(coll: Ident): ConnectionIO[InsightData] = { def getInsights(coll: Ident): ConnectionIO[InsightData] = {
val IC = RItem.Columns val IC = RItem.Columns
val AC = RAttachment.Columns val AC = RAttachment.Columns
val TC = RTag.Columns val TC = RTag.Columns
val RC = RTagItem.Columns val RC = RTagItem.Columns
val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)). val q0 = selectCount(
query[Int].unique IC.id,
val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)). RItem.table,
query[Int].unique and(IC.cid.is(coll), IC.incoming.is(Direction.incoming))
).query[Int].unique
val q1 = selectCount(
IC.id,
RItem.table,
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique
val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++ val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++ fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++
fr"WHERE" ++ IC.cid.is(coll) fr"WHERE" ++ IC.cid.is(coll)
val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++ val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
) ++
fr"FROM" ++ RTagItem.table ++ fr"r" ++ fr"FROM" ++ RTagItem.table ++ fr"r" ++
fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++ fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++
fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++

Some files were not shown because too many files have changed in this diff Show More