diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 757b24ae..05ed9347 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -27,38 +27,44 @@ trait BackendApp[F[_]] { object BackendApp { - def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] = + def create[F[_]: ConcurrentEffect]( + cfg: Config, + store: Store[F], + httpClientEc: ExecutionContext + ): Resource[F, BackendApp[F]] = for { - queue <- JobQueue(store) - loginImpl <- Login[F](store) - signupImpl <- OSignup[F](store) - collImpl <- OCollective[F](store) - sourceImpl <- OSource[F](store) - tagImpl <- OTag[F](store) - equipImpl <- OEquipment[F](store) - orgImpl <- OOrganization(store) - uploadImpl <- OUpload(store, queue, cfg, httpClientEc) - nodeImpl <- ONode(store) - jobImpl <- OJob(store, httpClientEc) - itemImpl <- OItem(store) + queue <- JobQueue(store) + loginImpl <- Login[F](store) + signupImpl <- OSignup[F](store) + collImpl <- OCollective[F](store) + sourceImpl <- OSource[F](store) + tagImpl <- OTag[F](store) + equipImpl <- OEquipment[F](store) + orgImpl <- OOrganization(store) + uploadImpl <- OUpload(store, queue, cfg, httpClientEc) + nodeImpl <- ONode(store) + jobImpl <- OJob(store, httpClientEc) + itemImpl <- OItem(store) } yield new BackendApp[F] { - val login: Login[F] = loginImpl - val signup: OSignup[F] = signupImpl + val login: Login[F] = loginImpl + val signup: OSignup[F] = signupImpl val collective: OCollective[F] = collImpl - val source = sourceImpl - val tag = tagImpl - val equipment = equipImpl - val organization = orgImpl - val upload = uploadImpl - val node = nodeImpl - val job = jobImpl - val item = itemImpl + val source = sourceImpl + val tag = tagImpl + val equipment = equipImpl + val organization = orgImpl + val upload = uploadImpl + val node = nodeImpl + val job = jobImpl + val item = itemImpl } - def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config - , connectEC: ExecutionContext - , httpClientEc: ExecutionContext - , blocker: Blocker): Resource[F, BackendApp[F]] = + def apply[F[_]: ConcurrentEffect: ContextShift]( + cfg: Config, + connectEC: ExecutionContext, + httpClientEc: ExecutionContext, + blocker: Blocker + ): Resource[F, BackendApp[F]] = for { store <- Store.create(cfg.jdbc, connectEC, blocker) backend <- create(cfg, store, httpClientEc) diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index 7869019c..ed0837bb 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -4,13 +4,9 @@ import docspell.backend.signup.{Config => SignupConfig} import docspell.common.MimeType import docspell.store.JdbcConfig -case class Config( jdbc: JdbcConfig - , signup: SignupConfig - , files: Config.Files) { - -} +case class Config(jdbc: JdbcConfig, signup: SignupConfig, files: Config.Files) {} object Config { case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) -} \ No newline at end of file +} diff --git a/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala b/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala index 57d3f5ca..98ace40c 100644 --- a/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala +++ b/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala @@ -50,14 +50,13 @@ object AuthToken { Left("Invalid authenticator") } - def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = { + def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = for { salt <- Common.genSaltString[F] millis = Instant.now.toEpochMilli - cd = AuthToken(millis, accountId, salt, "") - sig = sign(cd, key) + cd = AuthToken(millis, accountId, salt, "") + sig = sign(cd, key) } yield cd.copy(sig = sig) - } private def sign(cd: AuthToken, key: ByteVector): String = { val raw = cd.millis.toString + cd.account.asString + cd.salt diff --git a/modules/backend/src/main/scala/docspell/backend/auth/Login.scala b/modules/backend/src/main/scala/docspell/backend/auth/Login.scala index 5c75a8bf..55080ab2 100644 --- a/modules/backend/src/main/scala/docspell/backend/auth/Login.scala +++ b/modules/backend/src/main/scala/docspell/backend/auth/Login.scala @@ -45,45 +45,45 @@ object Login { } def ok(session: AuthToken): Result = Ok(session) - def invalidAuth: Result = InvalidAuth - def invalidTime: Result = InvalidTime + def invalidAuth: Result = InvalidAuth + def invalidTime: Result = InvalidTime } - def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] { + def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = + Resource.pure(new Login[F] { - def loginSession(config: Config)(sessionKey: String): F[Result] = - AuthToken.fromString(sessionKey) match { - case Right(at) => - if (at.sigInvalid(config.serverSecret)) Result.invalidAuth.pure[F] - else if (at.isExpired(config.sessionValid)) Result.invalidTime.pure[F] - else Result.ok(at).pure[F] - case Left(_) => - Result.invalidAuth.pure[F] + def loginSession(config: Config)(sessionKey: String): F[Result] = + AuthToken.fromString(sessionKey) match { + case Right(at) => + if (at.sigInvalid(config.serverSecret)) Result.invalidAuth.pure[F] + else if (at.isExpired(config.sessionValid)) Result.invalidTime.pure[F] + else Result.ok(at).pure[F] + case Left(_) => + Result.invalidAuth.pure[F] + } + + def loginUserPass(config: Config)(up: UserPass): F[Result] = + AccountId.parse(up.user) match { + case Right(acc) => + val okResult = + store.transact(RUser.updateLogin(acc)) *> + AuthToken.user(acc, config.serverSecret).map(Result.ok) + for { + data <- store.transact(QLogin.findUser(acc)) + _ <- Sync[F].delay(logger.trace(s"Account lookup: $data")) + res <- if (data.exists(check(up.pass))) okResult + else Result.invalidAuth.pure[F] + } yield res + case Left(_) => + Result.invalidAuth.pure[F] + } + + private def check(given: String)(data: QLogin.Data): Boolean = { + val collOk = data.collectiveState == CollectiveState.Active || + data.collectiveState == CollectiveState.ReadOnly + val userOk = data.userState == UserState.Active + val passOk = BCrypt.checkpw(given, data.password.pass) + collOk && userOk && passOk } - - def loginUserPass(config: Config)(up: UserPass): F[Result] = { - AccountId.parse(up.user) match { - case Right(acc) => - val okResult= - store.transact(RUser.updateLogin(acc)) *> - AuthToken.user(acc, config.serverSecret).map(Result.ok) - for { - data <- store.transact(QLogin.findUser(acc)) - _ <- Sync[F].delay(logger.trace(s"Account lookup: $data")) - res <- if (data.exists(check(up.pass))) okResult - else Result.invalidAuth.pure[F] - } yield res - case Left(_) => - Result.invalidAuth.pure[F] - } - } - - private def check(given: String)(data: QLogin.Data): Boolean = { - val collOk = data.collectiveState == CollectiveState.Active || - data.collectiveState == CollectiveState.ReadOnly - val userOk = data.userState == UserState.Active - val passOk = BCrypt.checkpw(given, data.password.pass) - collOk && userOk && passOk - } - }) + }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index 67fe0736..90099c0a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -25,7 +25,11 @@ trait OCollective[F[_]] { def insights(collective: Ident): F[InsightData] - def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] + def changePassword( + accountId: AccountId, + current: Password, + newPass: Password + ): F[PassChangeResult] } object OCollective { @@ -35,15 +39,15 @@ object OCollective { sealed trait PassChangeResult object PassChangeResult { - case object UserNotFound extends PassChangeResult + case object UserNotFound extends PassChangeResult case object PasswordMismatch extends PassChangeResult - case object UpdateFailed extends PassChangeResult - case object Success extends PassChangeResult + case object UpdateFailed extends PassChangeResult + case object Success extends PassChangeResult - def userNotFound: PassChangeResult = UserNotFound + def userNotFound: PassChangeResult = UserNotFound def passwordMismatch: PassChangeResult = PasswordMismatch - def success: PassChangeResult = Success - def updateFailed: PassChangeResult = UpdateFailed + def success: PassChangeResult = Success + def updateFailed: PassChangeResult = UpdateFailed } case class RegisterData(collName: Ident, login: Ident, password: Password, invite: Option[Ident]) @@ -63,39 +67,47 @@ object OCollective { } } - - def apply[F[_]:Effect](store: Store[F]): Resource[F, OCollective[F]] = + def apply[F[_]: Effect](store: Store[F]): Resource[F, OCollective[F]] = Resource.pure(new OCollective[F] { def find(name: Ident): F[Option[RCollective]] = store.transact(RCollective.findById(name)) def updateLanguage(collective: Ident, lang: Language): F[AddResult] = - store.transact(RCollective.updateLanguage(collective, lang)). - attempt.map(AddResult.fromUpdate) + store + .transact(RCollective.updateLanguage(collective, lang)) + .attempt + .map(AddResult.fromUpdate) - def listUser(collective: Ident): F[Vector[RUser]] = { + def listUser(collective: Ident): F[Vector[RUser]] = store.transact(RUser.findAll(collective, _.login)) - } def add(s: RUser): F[AddResult] = - store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login)) + store.add( + RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), + RUser.exists(s.login) + ) def update(s: RUser): F[AddResult] = store.add(RUser.update(s), RUser.exists(s.login)) def deleteUser(login: Ident, collective: Ident): F[AddResult] = - store.transact(RUser.delete(login, collective)). - attempt.map(AddResult.fromUpdate) + store.transact(RUser.delete(login, collective)).attempt.map(AddResult.fromUpdate) def insights(collective: Ident): F[InsightData] = store.transact(QCollective.getInsights(collective)) - def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = { + def changePassword( + accountId: AccountId, + current: Password, + newPass: Password + ): F[PassChangeResult] = { val q = for { optUser <- RUser.findByAccount(accountId) - check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) - n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) - res = check match { + check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) + n <- check + .filter(identity) + .traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) + res = check match { case Some(true) => if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed case Some(false) => diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala b/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala index 8ac4698f..279b8db3 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala @@ -17,7 +17,6 @@ trait OEquipment[F[_]] { def delete(id: Ident, collective: Ident): F[AddResult] } - object OEquipment { def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] = @@ -43,12 +42,10 @@ object OEquipment { def delete(id: Ident, collective: Ident): F[AddResult] = { val io = for { - n0 <- RItem.removeConcEquip(collective, id) - n1 <- REquipment.delete(id, collective) + n0 <- RItem.removeConcEquip(collective, id) + n1 <- REquipment.delete(id, collective) } yield n0 + n1 - store.transact(io). - attempt. - map(AddResult.fromUpdate) + store.transact(io).attempt.map(AddResult.fromUpdate) } }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 4db3409f..72dd7e16 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -62,90 +62,98 @@ object OItem { case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte]) - def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] = Resource.pure(new OItem[F] { def findItem(id: Ident, collective: Ident): F[Option[ItemData]] = - store.transact(QItem.findItem(id)). - map(opt => opt.flatMap(_.filterCollective(collective))) + store.transact(QItem.findItem(id)).map(opt => opt.flatMap(_.filterCollective(collective))) - def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = { + def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector - } - def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = { - store.transact(RAttachment.findByIdAndCollective(id, collective)). - flatMap({ + def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = + store + .transact(RAttachment.findByIdAndCollective(id, collective)) + .flatMap({ case Some(ra) => - store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last. - map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))))) + store.bitpeace + .get(ra.fileId.id) + .unNoneTerminate + .compile + .last + .map( + _.map(m => + AttachmentData[F]( + ra, + m, + store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m)) + ) + ) + ) case None => (None: Option[AttachmentData[F]]).pure[F] }) - } def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = { val db = for { cid <- RItem.getCollective(item) - nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO] - ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO] + nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) + else 0.pure[ConnectionIO] + ni <- if (tagIds.nonEmpty && cid.contains(collective)) + RTagItem.insertItemTags(item, tagIds) + else 0.pure[ConnectionIO] } yield nd + ni - store.transact(db). - attempt. - map(AddResult.fromUpdate) + store.transact(db).attempt.map(AddResult.fromUpdate) } def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] = - store.transact(RItem.updateDirection(item, collective, direction)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateDirection(item, collective, direction)) + .attempt + .map(AddResult.fromUpdate) def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] = - store.transact(RItem.updateCorrOrg(item, collective, org)). - attempt. - map(AddResult.fromUpdate) + store.transact(RItem.updateCorrOrg(item, collective, org)).attempt.map(AddResult.fromUpdate) def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = - store.transact(RItem.updateCorrPerson(item, collective, person)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateCorrPerson(item, collective, person)) + .attempt + .map(AddResult.fromUpdate) def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = - store.transact(RItem.updateConcPerson(item, collective, person)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateConcPerson(item, collective, person)) + .attempt + .map(AddResult.fromUpdate) def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] = - store.transact(RItem.updateConcEquip(item, collective, equip)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateConcEquip(item, collective, equip)) + .attempt + .map(AddResult.fromUpdate) def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] = - store.transact(RItem.updateNotes(item, collective, notes)). - attempt. - map(AddResult.fromUpdate) + store.transact(RItem.updateNotes(item, collective, notes)).attempt.map(AddResult.fromUpdate) def setName(item: Ident, name: String, collective: Ident): F[AddResult] = - store.transact(RItem.updateName(item, collective, name)). - attempt. - map(AddResult.fromUpdate) + store.transact(RItem.updateName(item, collective, name)).attempt.map(AddResult.fromUpdate) def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] = - store.transact(RItem.updateStateForCollective(item, state, collective)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateStateForCollective(item, state, collective)) + .attempt + .map(AddResult.fromUpdate) def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = - store.transact(RItem.updateDate(item, collective, date)). - attempt. - map(AddResult.fromUpdate) + store.transact(RItem.updateDate(item, collective, date)).attempt.map(AddResult.fromUpdate) def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = - store.transact(RItem.updateDueDate(item, collective, date)). - attempt. - map(AddResult.fromUpdate) + store + .transact(RItem.updateDueDate(item, collective, date)) + .attempt + .map(AddResult.fromUpdate) def delete(itemId: Ident, collective: Ident): F[Int] = QItem.delete(store)(itemId, collective) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala index 189123de..3a12d4bf 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala @@ -21,9 +21,9 @@ object OJob { sealed trait JobCancelResult object JobCancelResult { - case object Removed extends JobCancelResult + case object Removed extends JobCancelResult case object CancelRequested extends JobCancelResult - case object JobNotFound extends JobCancelResult + case object JobNotFound extends JobCancelResult } case class JobDetail(job: RJob, logs: Vector[RJobLog]) @@ -36,15 +36,19 @@ object OJob { jobs.filter(_.job.state == JobState.Running) } - def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] = + def apply[F[_]: ConcurrentEffect]( + store: Store[F], + clientEC: ExecutionContext + ): Resource[F, OJob[F]] = Resource.pure(new OJob[F] { - def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = { - store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)). - map(t => JobDetail(t._1, t._2)). - compile.toVector. - map(CollectiveQueueState) - } + def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = + store + .transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)) + .map(t => JobDetail(t._1, t._2)) + .compile + .toVector + .map(CollectiveQueueState) def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = { def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] = @@ -58,26 +62,27 @@ object OJob { val tryDelete = for { job <- RJob.findByIdAndGroup(id, collective) - jobm = job.filter(canDelete) + jobm = job.filter(canDelete) del <- jobm.traverse(j => RJob.delete(j.id)) } yield del match { case Some(_) => Right(JobCancelResult.Removed: JobCancelResult) - case None => Left(mustCancel(job)) + case None => Left(mustCancel(job)) } def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] = - OJoex.cancelJob(job.id, worker, store, clientEC). - map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound) + OJoex + .cancelJob(job.id, worker, store, clientEC) + .map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound) for { - tryDel <- store.transact(tryDelete) - result <- tryDel match { - case Right(r) => r.pure[F] - case Left(Some((job, worker))) => - tryCancel(job, worker) - case Left(None) => - (JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F] - } + tryDel <- store.transact(tryDelete) + result <- tryDel match { + case Right(r) => r.pure[F] + case Left(Some((job, worker))) => + tryCancel(job, worker) + case Left(None) => + (JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F] + } } yield result } }) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index 6c120ee1..151b8485 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -13,24 +13,32 @@ import scala.concurrent.ExecutionContext import org.log4s._ object OJoex { - private [this] val logger = getLogger + private[this] val logger = getLogger - def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = { + def notifyAll[F[_]: ConcurrentEffect]( + store: Store[F], + clientExecutionContext: ExecutionContext + ): F[Unit] = for { nodes <- store.transact(RNode.findAll(NodeType.Joex)) _ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext)) } yield () - } - def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] = + def cancelJob[F[_]: ConcurrentEffect]( + jobId: Ident, + worker: Ident, + store: Store[F], + clientEc: ExecutionContext + ): F[Boolean] = for { - node <- store.transact(RNode.findById(worker)) + node <- store.transact(RNode.findById(worker)) cancel <- node.traverse(joexCancel(clientEc)(_, jobId)) } yield cancel.getOrElse(false) - - private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = { - val notifyUrl = node.url/"api"/"v1"/"job"/job.id/"cancel" + private def joexCancel[F[_]: ConcurrentEffect]( + ec: ExecutionContext + )(node: RNode, job: Ident): F[Boolean] = { + val notifyUrl = node.url / "api" / "v1" / "job" / job.id / "cancel" BlazeClientBuilder[F](ec).resource.use { client => val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) client.expect[String](req).map(_ => true) @@ -38,7 +46,7 @@ object OJoex { } private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = { - val notifyUrl = node.url/"api"/"v1"/"notify" + val notifyUrl = node.url / "api" / "v1" / "notify" val execute = BlazeClientBuilder[F](ec).resource.use { client => val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) client.expect[String](req).map(_ => ()) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala b/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala index 356d7a56..5bf7cd11 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala @@ -36,13 +36,15 @@ object OOrganization { case class PersonAndContacts(person: RPerson, contacts: Seq[RContact]) - def apply[F[_] : Effect](store: Store[F]): Resource[F, OOrganization[F]] = + def apply[F[_]: Effect](store: Store[F]): Resource[F, OOrganization[F]] = Resource.pure(new OOrganization[F] { def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] = - store.transact(QOrganization.findOrgAndContact(account.collective, _.name)). - map({ case (org, cont) => OrgAndContacts(org, cont) }). - compile.toVector + store + .transact(QOrganization.findOrgAndContact(account.collective, _.name)) + .map({ case (org, cont) => OrgAndContacts(org, cont) }) + .compile + .toVector def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] = store.transact(ROrganization.findAllRef(account.collective, _.name)) @@ -54,9 +56,11 @@ object OOrganization { QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store) def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] = - store.transact(QOrganization.findPersonAndContact(account.collective, _.name)). - map({ case (person, cont) => PersonAndContacts(person, cont) }). - compile.toVector + store + .transact(QOrganization.findPersonAndContact(account.collective, _.name)) + .map({ case (person, cont) => PersonAndContacts(person, cont) }) + .compile + .toVector def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] = store.transact(RPerson.findAllRef(account.collective, _.name)) @@ -68,14 +72,13 @@ object OOrganization { QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store) def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] = - store.transact(QOrganization.deleteOrg(orgId, collective)). - attempt. - map(AddResult.fromUpdate) + store.transact(QOrganization.deleteOrg(orgId, collective)).attempt.map(AddResult.fromUpdate) def deletePerson(personId: Ident, collective: Ident): F[AddResult] = - store.transact(QOrganization.deletePerson(personId, collective)). - attempt. - map(AddResult.fromUpdate) + store + .transact(QOrganization.deletePerson(personId, collective)) + .attempt + .map(AddResult.fromUpdate) }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala index 227ae752..9f3e685d 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala @@ -41,8 +41,6 @@ object OSource { } def delete(id: Ident, collective: Ident): F[AddResult] = - store.transact(RSource.delete(id, collective)). - attempt. - map(AddResult.fromUpdate) + store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate) }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala b/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala index 54ccec58..1414c3fc 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala @@ -17,7 +17,6 @@ trait OTag[F[_]] { def delete(id: Ident, collective: Ident): F[AddResult] } - object OTag { def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] = @@ -47,10 +46,7 @@ object OTag { n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId)) n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective)) } yield n0.getOrElse(0) + n1.getOrElse(0) - store.transact(io). - attempt. - map(AddResult.fromUpdate) + store.transact(io).attempt.map(AddResult.fromUpdate) } }) } - diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index 0351c1d9..07be67b6 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -22,75 +22,113 @@ trait OUpload[F[_]] { } object OUpload { - private [this] val logger = getLogger + private[this] val logger = getLogger - case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte]) + case class File[F[_]]( + name: Option[String], + advertisedMime: Option[MimeType], + data: Stream[F, Byte] + ) - case class UploadMeta( direction: Option[Direction] - , sourceAbbrev: String - , validFileTypes: Seq[MimeType]) + case class UploadMeta( + direction: Option[Direction], + sourceAbbrev: String, + validFileTypes: Seq[MimeType] + ) - case class UploadData[F[_]]( multiple: Boolean - , meta: UploadMeta - , files: Vector[File[F]], priority: Priority, tracker: Option[Ident]) + case class UploadData[F[_]]( + multiple: Boolean, + meta: UploadMeta, + files: Vector[File[F]], + priority: Priority, + tracker: Option[Ident] + ) sealed trait UploadResult object UploadResult { - case object Success extends UploadResult - case object NoFiles extends UploadResult + case object Success extends UploadResult + case object NoFiles extends UploadResult case object NoSource extends UploadResult } - def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] = + def apply[F[_]: ConcurrentEffect]( + store: Store[F], + queue: JobQueue[F], + cfg: Config, + httpClientEC: ExecutionContext + ): Resource[F, OUpload[F]] = Resource.pure(new OUpload[F] { - def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = { + def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = for { files <- data.files.traverse(saveFile).map(_.flatten) pred <- checkFileList(files) lang <- store.transact(RCollective.findLanguage(account.collective)) - meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes) - args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList)) - job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker)) - _ <- logger.fdebug(s"Storing jobs: $job") - res <- job.traverse(submitJobs) - _ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)) + meta = ProcessItemArgs.ProcessMeta( + account.collective, + lang.getOrElse(Language.German), + data.meta.direction, + data.meta.sourceAbbrev, + data.meta.validFileTypes + ) + args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) + else Vector(ProcessItemArgs(meta, files.toList)) + job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker)) + _ <- logger.fdebug(s"Storing jobs: $job") + res <- job.traverse(submitJobs) + _ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)) } yield res.fold(identity, identity) - } def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] = for { - sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource)) - abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev) - updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev)) - accId = sOpt.map(source => AccountId(source.cid, source.sid)) - result <- accId.traverse(acc => submit(updata, acc)) + sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource)) + abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev) + updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev)) + accId = sOpt.map(source => AccountId(source.cid, source.sid)) + result <- accId.traverse(acc => submit(updata, acc)) } yield result.fold(identity, identity) - private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = { + private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = for { - _ <- logger.fdebug(s"Storing jobs: $jobs") - _ <- queue.insertAll(jobs) - _ <- OJoex.notifyAll(store, httpClientEC) + _ <- logger.fdebug(s"Storing jobs: $jobs") + _ <- queue.insertAll(jobs) + _ <- OJoex.notifyAll(store, httpClientEC) } yield UploadResult.Success - } - private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = { + private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = logger.finfo(s"Receiving file $file") *> - store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None). - compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt. - map(_.fold(ex => { - logger.warn(ex)(s"Could not store file for processing!") - None - }, id => Some(ProcessItemArgs.File(file.name, id)))) - } + store.bitpeace + .saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None) + .compile + .lastOrError + .map(fm => Ident.unsafe(fm.id)) + .attempt + .map(_.fold(ex => { + logger.warn(ex)(s"Could not store file for processing!") + None + }, id => Some(ProcessItemArgs.File(file.name, id)))) private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] = Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(())) - private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = { + private def makeJobs( + args: Vector[ProcessItemArgs], + account: AccountId, + prio: Priority, + tracker: Option[Ident] + ): F[Vector[RJob]] = { def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob = - RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker) + RJob.newJob( + id, + ProcessItemArgs.taskName, + account.collective, + arg, + arg.makeSubject, + now, + account.user, + prio, + tracker + ) for { id <- Ident.randomId[F] diff --git a/modules/backend/src/main/scala/docspell/backend/signup/Config.scala b/modules/backend/src/main/scala/docspell/backend/signup/Config.scala index f1c053e8..e7b9b1dd 100644 --- a/modules/backend/src/main/scala/docspell/backend/signup/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/signup/Config.scala @@ -20,10 +20,10 @@ object Config { def fromString(str: String): Either[String, Mode] = str.toLowerCase match { - case "open" => Right(Open) + case "open" => Right(Open) case "invite" => Right(Invite) case "closed" => Right(Closed) - case _ => Left(s"Invalid signup mode: $str") + case _ => Left(s"Invalid signup mode: $str") } def unsafe(str: String): Mode = fromString(str).fold(sys.error, identity) @@ -34,7 +34,7 @@ object Config { Decoder.decodeString.emap(fromString) } - def open: Mode = Mode.Open + def open: Mode = Mode.Open def invite: Mode = Mode.Invite def closed: Mode = Mode.Closed diff --git a/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala b/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala index b963d4a6..88b7d59a 100644 --- a/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala +++ b/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala @@ -9,11 +9,11 @@ sealed trait NewInviteResult { self: Product => } object NewInviteResult { - case class Success(id: Ident) extends NewInviteResult + case class Success(id: Ident) extends NewInviteResult case object InvitationDisabled extends NewInviteResult - case object PasswordMismatch extends NewInviteResult + case object PasswordMismatch extends NewInviteResult - def passwordMismatch: NewInviteResult = PasswordMismatch - def invitationClosed: NewInviteResult = InvitationDisabled + def passwordMismatch: NewInviteResult = PasswordMismatch + def invitationClosed: NewInviteResult = InvitationDisabled def success(id: Ident): NewInviteResult = Success(id) } diff --git a/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala b/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala index 76b6264b..40159449 100644 --- a/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala +++ b/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala @@ -21,19 +21,19 @@ trait OSignup[F[_]] { object OSignup { private[this] val logger = getLogger - def apply[F[_]:Effect](store: Store[F]): Resource[F, OSignup[F]] = + def apply[F[_]: Effect](store: Store[F]): Resource[F, OSignup[F]] = Resource.pure(new OSignup[F] { - def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = { + def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = if (cfg.mode == Config.Mode.Invite) { - if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F] + if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) + NewInviteResult.passwordMismatch.pure[F] else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id)) } else { Effect[F].pure(NewInviteResult.invitationClosed) } - } - def register(cfg: Config)(data: RegisterData): F[SignupResult] = { + def register(cfg: Config)(data: RegisterData): F[SignupResult] = cfg.mode match { case Config.Mode.Open => addUser(data).map(SignupResult.fromAddResult) @@ -45,11 +45,11 @@ object OSignup { data.invite match { case Some(inv) => for { - now <- Timestamp.current[F] - min = now.minus(cfg.inviteTime) - ok <- store.transact(RInvitation.useInvite(inv, min)) - res <- if (ok) addUser(data).map(SignupResult.fromAddResult) - else SignupResult.invalidInvitationKey.pure[F] + now <- Timestamp.current[F] + min = now.minus(cfg.inviteTime) + ok <- store.transact(RInvitation.useInvite(inv, min)) + res <- if (ok) addUser(data).map(SignupResult.fromAddResult) + else SignupResult.invalidInvitationKey.pure[F] _ <- if (retryInvite(res)) logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store .transact( @@ -61,7 +61,6 @@ object OSignup { SignupResult.invalidInvitationKey.pure[F] } } - } private def retryInvite(res: SignupResult): Boolean = res match { @@ -77,29 +76,37 @@ object OSignup { false } - private def addUser(data: RegisterData): F[AddResult] = { def toRecords: F[(RCollective, RUser)] = for { id2 <- Ident.randomId[F] now <- Timestamp.current[F] - c = RCollective(data.collName, CollectiveState.Active, Language.German, now) - u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now) + c = RCollective(data.collName, CollectiveState.Active, Language.German, now) + u = RUser( + id2, + data.login, + data.collName, + PasswordCrypt.crypt(data.password), + UserState.Active, + None, + 0, + None, + now + ) } yield (c, u) - def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = { + def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = for { n1 <- RCollective.insert(coll) n2 <- RUser.insert(user) } yield n1 + n2 - } def collectiveExists: ConnectionIO[Boolean] = RCollective.existsById(data.collName) val msg = s"The collective '${data.collName}' already exists." for { - cu <- toRecords + cu <- toRecords save <- store.add(insert(cu._1, cu._2), collectiveExists) } yield save.fold(identity, _.withMsg(msg), identity) } diff --git a/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala b/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala index e8230059..15782324 100644 --- a/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala +++ b/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala @@ -2,27 +2,25 @@ package docspell.backend.signup import docspell.store.AddResult -sealed trait SignupResult { - -} +sealed trait SignupResult {} object SignupResult { - case object CollectiveExists extends SignupResult - case object InvalidInvitationKey extends SignupResult - case object SignupClosed extends SignupResult + case object CollectiveExists extends SignupResult + case object InvalidInvitationKey extends SignupResult + case object SignupClosed extends SignupResult case class Failure(ex: Throwable) extends SignupResult - case object Success extends SignupResult + case object Success extends SignupResult - def collectiveExists: SignupResult = CollectiveExists - def invalidInvitationKey: SignupResult = InvalidInvitationKey - def signupClosed: SignupResult = SignupClosed + def collectiveExists: SignupResult = CollectiveExists + def invalidInvitationKey: SignupResult = InvalidInvitationKey + def signupClosed: SignupResult = SignupClosed def failure(ex: Throwable): SignupResult = Failure(ex) - def success: SignupResult = Success + def success: SignupResult = Success def fromAddResult(ar: AddResult): SignupResult = ar match { - case AddResult.Success => Success - case AddResult.Failure(ex) => Failure(ex) + case AddResult.Success => Success + case AddResult.Failure(ex) => Failure(ex) case AddResult.EntityExists(_) => CollectiveExists } } diff --git a/modules/common/src/main/scala/docspell/common/Banner.scala b/modules/common/src/main/scala/docspell/common/Banner.scala index d84cd00d..8f8ccac3 100644 --- a/modules/common/src/main/scala/docspell/common/Banner.scala +++ b/modules/common/src/main/scala/docspell/common/Banner.scala @@ -1,12 +1,14 @@ package docspell.common -case class Banner( component: String - , version: String - , gitHash: Option[String] - , jdbcUrl: LenientUri - , configFile: Option[String] - , appId: Ident - , baseUrl: LenientUri) { +case class Banner( + component: String, + version: String, + gitHash: Option[String], + jdbcUrl: LenientUri, + configFile: Option[String], + appId: Ident, + baseUrl: LenientUri +) { private val banner = """______ _ _ @@ -17,16 +19,16 @@ case class Banner( component: String ||___/ \___/ \___|___/ .__/ \___|_|_| | | | |""".stripMargin + - s""" |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})""" + s""" |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})""" def render(prefix: String): String = { val text = banner.split('\n').toList ++ List( - s"<< $component >>" - , s"Id: ${appId.id}" - , s"Base-Url: ${baseUrl.asString}" - , s"Database: ${jdbcUrl.asString}" - , s"Config: ${configFile.getOrElse("")}" - , "" + s"<< $component >>", + s"Id: ${appId.id}", + s"Base-Url: ${baseUrl.asString}", + s"Database: ${jdbcUrl.asString}", + s"Config: ${configFile.getOrElse("")}", + "" ) text.map(line => s"$prefix $line").mkString("\n") diff --git a/modules/common/src/main/scala/docspell/common/Direction.scala b/modules/common/src/main/scala/docspell/common/Direction.scala index a96d8338..cb4bcb4c 100644 --- a/modules/common/src/main/scala/docspell/common/Direction.scala +++ b/modules/common/src/main/scala/docspell/common/Direction.scala @@ -21,7 +21,7 @@ object Direction { str.toLowerCase match { case "incoming" => Right(Incoming) case "outgoing" => Right(Outgoing) - case _ => Left(s"No direction: $str") + case _ => Left(s"No direction: $str") } def unsafe(str: String): Direction = diff --git a/modules/common/src/main/scala/docspell/common/IdRef.scala b/modules/common/src/main/scala/docspell/common/IdRef.scala index d33c8535..8c32405c 100644 --- a/modules/common/src/main/scala/docspell/common/IdRef.scala +++ b/modules/common/src/main/scala/docspell/common/IdRef.scala @@ -3,9 +3,7 @@ package docspell.common import io.circe._ import io.circe.generic.semiauto._ -case class IdRef(id: Ident, name: String) { - -} +case class IdRef(id: Ident, name: String) {} object IdRef { @@ -13,4 +11,4 @@ object IdRef { deriveEncoder[IdRef] implicit val jsonDecoder: Decoder[IdRef] = deriveDecoder[IdRef] -} \ No newline at end of file +} diff --git a/modules/common/src/main/scala/docspell/common/ItemState.scala b/modules/common/src/main/scala/docspell/common/ItemState.scala index 4f4375d2..ed4e5511 100644 --- a/modules/common/src/main/scala/docspell/common/ItemState.scala +++ b/modules/common/src/main/scala/docspell/common/ItemState.scala @@ -10,18 +10,18 @@ sealed trait ItemState { self: Product => object ItemState { - case object Premature extends ItemState + case object Premature extends ItemState case object Processing extends ItemState - case object Created extends ItemState - case object Confirmed extends ItemState + case object Created extends ItemState + case object Confirmed extends ItemState def fromString(str: String): Either[String, ItemState] = str.toLowerCase match { - case "premature" => Right(Premature) + case "premature" => Right(Premature) case "processing" => Right(Processing) - case "created" => Right(Created) - case "confirmed" => Right(Confirmed) - case _ => Left(s"Invalid item state: $str") + case "created" => Right(Created) + case "confirmed" => Right(Confirmed) + case _ => Left(s"Invalid item state: $str") } def unsafe(str: String): ItemState = @@ -32,4 +32,3 @@ object ItemState { implicit val jsonEncoder: Encoder[ItemState] = Encoder.encodeString.contramap(_.name) } - diff --git a/modules/common/src/main/scala/docspell/common/Language.scala b/modules/common/src/main/scala/docspell/common/Language.scala index f82b3622..7d836347 100644 --- a/modules/common/src/main/scala/docspell/common/Language.scala +++ b/modules/common/src/main/scala/docspell/common/Language.scala @@ -31,14 +31,12 @@ object Language { def fromString(str: String): Either[String, Language] = { val lang = str.toLowerCase - all.find(_.allNames.contains(lang)). - toRight(s"Unsupported or invalid language: $str") + all.find(_.allNames.contains(lang)).toRight(s"Unsupported or invalid language: $str") } def unsafe(str: String): Language = fromString(str).fold(sys.error, identity) - implicit val jsonDecoder: Decoder[Language] = Decoder.decodeString.emap(fromString) implicit val jsonEncoder: Encoder[Language] = diff --git a/modules/common/src/main/scala/docspell/common/LogLevel.scala b/modules/common/src/main/scala/docspell/common/LogLevel.scala index 6cd5cbc8..d2404d98 100644 --- a/modules/common/src/main/scala/docspell/common/LogLevel.scala +++ b/modules/common/src/main/scala/docspell/common/LogLevel.scala @@ -11,8 +11,8 @@ sealed trait LogLevel { self: Product => object LogLevel { case object Debug extends LogLevel { val toInt = 0 } - case object Info extends LogLevel { val toInt = 1 } - case object Warn extends LogLevel { val toInt = 2 } + case object Info extends LogLevel { val toInt = 1 } + case object Warn extends LogLevel { val toInt = 2 } case object Error extends LogLevel { val toInt = 3 } def fromInt(n: Int): LogLevel = @@ -26,12 +26,12 @@ object LogLevel { def fromString(str: String): Either[String, LogLevel] = str.toLowerCase match { - case "debug" => Right(Debug) - case "info" => Right(Info) - case "warn" => Right(Warn) + case "debug" => Right(Debug) + case "info" => Right(Info) + case "warn" => Right(Warn) case "warning" => Right(Warn) - case "error" => Right(Error) - case _ => Left(s"Invalid log-level: $str") + case "error" => Right(Error) + case _ => Left(s"Invalid log-level: $str") } def unsafeString(str: String): LogLevel = @@ -41,4 +41,4 @@ object LogLevel { Decoder.decodeString.emap(fromString) implicit val jsonEncoder: Encoder[LogLevel] = Encoder.encodeString.contramap(_.name) -} \ No newline at end of file +} diff --git a/modules/common/src/main/scala/docspell/common/MimeType.scala b/modules/common/src/main/scala/docspell/common/MimeType.scala index b111dd96..7e6e6647 100644 --- a/modules/common/src/main/scala/docspell/common/MimeType.scala +++ b/modules/common/src/main/scala/docspell/common/MimeType.scala @@ -12,7 +12,7 @@ case class MimeType(primary: String, sub: String) { def matches(other: MimeType): Boolean = primary == other.primary && - (sub == other.sub || sub == "*" ) + (sub == other.sub || sub == "*") } object MimeType { @@ -26,9 +26,10 @@ object MimeType { def image(sub: String): MimeType = MimeType("image", partFromString(sub).throwLeft) - private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet + private[this] val validChars: Set[Char] = + (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet - def parse(str: String): Either[String, MimeType] = { + def parse(str: String): Either[String, MimeType] = str.indexOf('/') match { case -1 => Left(s"Invalid MIME type: $str") case n => @@ -37,7 +38,6 @@ object MimeType { sub <- partFromString(str.substring(n + 1)) } yield MimeType(prim.toLowerCase, sub.toLowerCase) } - } def unsafe(str: String): MimeType = parse(str).throwLeft @@ -47,12 +47,12 @@ object MimeType { else Left(s"Invalid identifier: $s. Allowed chars: ${validChars.mkString}") val octetStream = application("octet-stream") - val pdf = application("pdf") - val png = image("png") - val jpeg = image("jpeg") - val tiff = image("tiff") - val html = text("html") - val plain = text("plain") + val pdf = application("pdf") + val png = image("png") + val jpeg = image("jpeg") + val tiff = image("tiff") + val html = text("html") + val plain = text("plain") implicit val jsonEncoder: Encoder[MimeType] = Encoder.encodeString.contramap(_.asString) diff --git a/modules/common/src/main/scala/docspell/common/NerDateLabel.scala b/modules/common/src/main/scala/docspell/common/NerDateLabel.scala index 3e350e62..960559c3 100644 --- a/modules/common/src/main/scala/docspell/common/NerDateLabel.scala +++ b/modules/common/src/main/scala/docspell/common/NerDateLabel.scala @@ -2,6 +2,4 @@ package docspell.common import java.time.LocalDate -case class NerDateLabel(date: LocalDate, label: NerLabel) { - -} +case class NerDateLabel(date: LocalDate, label: NerLabel) {} diff --git a/modules/common/src/main/scala/docspell/common/NerLabel.scala b/modules/common/src/main/scala/docspell/common/NerLabel.scala index 27ee7a62..7eddfc2d 100644 --- a/modules/common/src/main/scala/docspell/common/NerLabel.scala +++ b/modules/common/src/main/scala/docspell/common/NerLabel.scala @@ -3,9 +3,7 @@ package docspell.common import io.circe.generic.semiauto._ import io.circe.{Decoder, Encoder} -case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) { - -} +case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {} object NerLabel { implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel] diff --git a/modules/common/src/main/scala/docspell/common/NodeType.scala b/modules/common/src/main/scala/docspell/common/NodeType.scala index b060f100..3020ffb4 100644 --- a/modules/common/src/main/scala/docspell/common/NodeType.scala +++ b/modules/common/src/main/scala/docspell/common/NodeType.scala @@ -10,13 +10,13 @@ sealed trait NodeType { self: Product => object NodeType { case object Restserver extends NodeType - case object Joex extends NodeType + case object Joex extends NodeType def fromString(str: String): Either[String, NodeType] = str.toLowerCase match { case "restserver" => Right(Restserver) - case "joex" => Right(Joex) - case _ => Left(s"Invalid node type: $str") + case "joex" => Right(Joex) + case _ => Left(s"Invalid node type: $str") } def unsafe(str: String): NodeType = diff --git a/modules/common/src/main/scala/docspell/common/Password.scala b/modules/common/src/main/scala/docspell/common/Password.scala index 73ba9b02..88a9d09d 100644 --- a/modules/common/src/main/scala/docspell/common/Password.scala +++ b/modules/common/src/main/scala/docspell/common/Password.scala @@ -4,7 +4,7 @@ import io.circe.{Decoder, Encoder} final class Password(val pass: String) extends AnyVal { - def isEmpty: Boolean= pass.isEmpty + def isEmpty: Boolean = pass.isEmpty override def toString: String = if (pass.isEmpty) "" else "***" diff --git a/modules/common/src/main/scala/docspell/common/Priority.scala b/modules/common/src/main/scala/docspell/common/Priority.scala index 9d1db712..25f6f1c6 100644 --- a/modules/common/src/main/scala/docspell/common/Priority.scala +++ b/modules/common/src/main/scala/docspell/common/Priority.scala @@ -16,25 +16,23 @@ object Priority { case object Low extends Priority - def fromString(str: String): Either[String, Priority] = str.toLowerCase match { case "high" => Right(High) - case "low" => Right(Low) - case _ => Left(s"Invalid priority: $str") + case "low" => Right(Low) + case _ => Left(s"Invalid priority: $str") } def unsafe(str: String): Priority = fromString(str).fold(sys.error, identity) - def fromInt(n: Int): Priority = if (n <= toInt(Low)) Low else High def toInt(p: Priority): Int = p match { - case Low => 0 + case Low => 0 case High => 10 } diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala index c46164ab..73e7a951 100644 --- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -6,14 +6,13 @@ import ProcessItemArgs._ case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { - def makeSubject: String = { + def makeSubject: String = files.flatMap(_.name) match { - case Nil => s"${meta.sourceAbbrev}: No files" - case n :: Nil => n + case Nil => s"${meta.sourceAbbrev}: No files" + case n :: Nil => n case n1 :: n2 :: Nil => s"$n1, $n2" - case _ => s"${files.size} files from ${meta.sourceAbbrev}" + case _ => s"${files.size} files from ${meta.sourceAbbrev}" } - } } @@ -21,11 +20,13 @@ object ProcessItemArgs { val taskName = Ident.unsafe("process-item") - case class ProcessMeta( collective: Ident - , language: Language - , direction: Option[Direction] - , sourceAbbrev: String - , validFileTypes: Seq[MimeType]) + case class ProcessMeta( + collective: Ident, + language: Language, + direction: Option[Direction], + sourceAbbrev: String, + validFileTypes: Seq[MimeType] + ) object ProcessMeta { implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta] diff --git a/modules/common/src/main/scala/docspell/common/Timestamp.scala b/modules/common/src/main/scala/docspell/common/Timestamp.scala index b35d5210..4bafba64 100644 --- a/modules/common/src/main/scala/docspell/common/Timestamp.scala +++ b/modules/common/src/main/scala/docspell/common/Timestamp.scala @@ -30,9 +30,7 @@ object Timestamp { def current[F[_]: Sync]: F[Timestamp] = Sync[F].delay(Timestamp(Instant.now)) - - - implicit val encodeTimestamp: Encoder[Timestamp] = + implicit val encodeTimestamp: Encoder[Timestamp] = BaseJsonCodecs.encodeInstantEpoch.contramap(_.value) implicit val decodeTimestamp: Decoder[Timestamp] = diff --git a/modules/common/src/main/scala/docspell/common/UserState.scala b/modules/common/src/main/scala/docspell/common/UserState.scala index 9424a46c..81e6b4cb 100644 --- a/modules/common/src/main/scala/docspell/common/UserState.scala +++ b/modules/common/src/main/scala/docspell/common/UserState.scala @@ -12,19 +12,18 @@ object UserState { /** The user is blocked by an admin. */ case object Disabled extends UserState - def fromString(s: String): Either[String, UserState] = s.toLowerCase match { - case "active" => Right(Active) + case "active" => Right(Active) case "disabled" => Right(Disabled) - case _ => Left(s"Not a state value: $s") + case _ => Left(s"Not a state value: $s") } def unsafe(str: String): UserState = fromString(str).fold(sys.error, identity) def asString(s: UserState): String = s match { - case Active => "active" + case Active => "active" case Disabled => "disabled" } @@ -34,4 +33,4 @@ object UserState { implicit val userStateDecoder: Decoder[UserState] = Decoder.decodeString.emap(UserState.fromString) -} \ No newline at end of file +} diff --git a/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala index 282f9c32..fff6e7d8 100644 --- a/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala +++ b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala @@ -4,18 +4,18 @@ trait EitherSyntax { implicit final class LeftStringEitherOps[A](e: Either[String, A]) { def throwLeft: A = e match { - case Right(a) => a + case Right(a) => a case Left(err) => sys.error(err) } } implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) { def throwLeft: A = e match { - case Right(a) => a + case Right(a) => a case Left(err) => throw err } } } -object EitherSyntax extends EitherSyntax \ No newline at end of file +object EitherSyntax extends EitherSyntax diff --git a/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala index 58f6eb26..a752bce4 100644 --- a/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala +++ b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala @@ -11,13 +11,18 @@ trait StreamSyntax { implicit class StringStreamOps[F[_]](s: Stream[F, String]) { def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] = - s.fold("")(_ + _). - compile.last. - map(optStr => for { - str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value")) - json <- parse(str).leftMap(_.underlying) - value <- json.as[A] - } yield value) + s.fold("")(_ + _) + .compile + .last + .map(optStr => + for { + str <- optStr + .map(_.trim) + .toRight(new Exception("Empty string cannot be parsed into a value")) + json <- parse(str).leftMap(_.underlying) + value <- json.as[A] + } yield value + ) } diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index 037f6cbf..b8f6b7ff 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -5,17 +5,23 @@ import docspell.joex.scheduler.SchedulerConfig import docspell.store.JdbcConfig import docspell.text.ocr.{Config => OcrConfig} -case class Config(appId: Ident - , baseUrl: LenientUri - , bind: Config.Bind - , jdbc: JdbcConfig - , scheduler: SchedulerConfig - , extraction: OcrConfig +case class Config( + appId: Ident, + baseUrl: LenientUri, + bind: Config.Bind, + jdbc: JdbcConfig, + scheduler: SchedulerConfig, + extraction: OcrConfig ) object Config { - val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") - val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") + val postgres = + JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") + val h2 = JdbcConfig( + LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), + "sa", + "" + ) case class Bind(address: String, port: Int) } diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index 8ed1e1d2..e399a7c9 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -11,7 +11,6 @@ object ConfigFile { def loadConfig: Config = ConfigSource.default.at("docspell.joex").loadOrThrow[Config] - object Implicits { implicit val countingSchemeReader: ConfigReader[CountingScheme] = ConfigReader[String].emap(reason(CountingScheme.readString)) diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 8d0c6551..374cf396 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -12,11 +12,13 @@ import fs2.concurrent.SignallingRef import scala.concurrent.ExecutionContext -final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config - , nodeOps: ONode[F] - , store: Store[F] - , termSignal: SignallingRef[F, Boolean] - , val scheduler: Scheduler[F]) extends JoexApp[F] { +final class JoexAppImpl[F[_]: ConcurrentEffect: ContextShift: Timer]( + cfg: Config, + nodeOps: ONode[F], + store: Store[F], + termSignal: SignallingRef[F, Boolean], + val scheduler: Scheduler[F] +) extends JoexApp[F] { def init: F[Unit] = { val run = scheduler.start.compile.drain @@ -40,17 +42,25 @@ final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Conf object JoexAppImpl { - def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config - , termSignal: SignallingRef[F, Boolean] - , connectEC: ExecutionContext - , blocker: Blocker): Resource[F, JoexApp[F]] = + def create[F[_]: ConcurrentEffect: ContextShift: Timer]( + cfg: Config, + termSignal: SignallingRef[F, Boolean], + connectEC: ExecutionContext, + blocker: Blocker + ): Resource[F, JoexApp[F]] = for { - store <- Store.create(cfg.jdbc, connectEC, blocker) + store <- Store.create(cfg.jdbc, connectEC, blocker) nodeOps <- ONode(store) - sch <- SchedulerBuilder(cfg.scheduler, blocker, store). - withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])). - resource - app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) - appR <- Resource.make(app.init.map(_ => app))(_.shutdown) + sch <- SchedulerBuilder(cfg.scheduler, blocker, store) + .withTask( + JobTask.json( + ProcessItemArgs.taskName, + ItemHandler[F](cfg.extraction), + ItemHandler.onCancel[F] + ) + ) + .resource + app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) + appR <- Resource.make(app.init.map(_ => app))(_.shutdown) } yield appR } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 3dd58806..00919923 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -15,20 +15,26 @@ import scala.concurrent.ExecutionContext object JoexServer { + private case class App[F[_]]( + httpApp: HttpApp[F], + termSig: SignallingRef[F, Boolean], + exitRef: Ref[F, ExitCode] + ) - private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode]) - - def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker) - (implicit T: Timer[F]): Stream[F, Nothing] = { + def stream[F[_]: ConcurrentEffect: ContextShift]( + cfg: Config, + connectEC: ExecutionContext, + blocker: Blocker + )(implicit T: Timer[F]): Stream[F, Nothing] = { val app = for { - signal <- Resource.liftF(SignallingRef[F, Boolean](false)) + signal <- Resource.liftF(SignallingRef[F, Boolean](false)) exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success)) joexApp <- JoexAppImpl.create[F](cfg, signal, connectEC, blocker) httpApp = Router( "/api/info" -> InfoRoutes(), - "/api/v1" -> JoexRoutes(joexApp) + "/api/v1" -> JoexRoutes(joexApp) ).orNotFound // With Middlewares in place @@ -36,14 +42,15 @@ object JoexServer { } yield App(finalHttpApp, signal, exitCode) - - Stream.resource(app).flatMap(app => - BlazeServerBuilder[F]. - bindHttp(cfg.bind.port, cfg.bind.address). - withHttpApp(app.httpApp). - withoutBanner. - serveWhile(app.termSig, app.exitRef) - ) + Stream + .resource(app) + .flatMap(app => + BlazeServerBuilder[F] + .bindHttp(cfg.bind.port, cfg.bind.address) + .withHttpApp(app.httpApp) + .withoutBanner + .serveWhile(app.termSig, app.exitRef) + ) }.drain } diff --git a/modules/joex/src/main/scala/docspell/joex/Main.scala b/modules/joex/src/main/scala/docspell/joex/Main.scala index f442652b..a800acdb 100644 --- a/modules/joex/src/main/scala/docspell/joex/Main.scala +++ b/modules/joex/src/main/scala/docspell/joex/Main.scala @@ -14,10 +14,12 @@ object Main extends IOApp { private[this] val logger = getLogger val blockingEc: ExecutionContext = ExecutionContext.fromExecutor( - Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))) + Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking")) + ) val blocker = Blocker.liftExecutionContext(blockingEc) val connectEC: ExecutionContext = ExecutionContext.fromExecutorService( - Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))) + Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect")) + ) def run(args: List[String]) = { args match { @@ -40,12 +42,15 @@ object Main extends IOApp { } val cfg = ConfigFile.loadConfig - val banner = Banner("JOEX" - , BuildInfo.version - , BuildInfo.gitHeadCommit - , cfg.jdbc.url - , Option(System.getProperty("config.file")) - , cfg.appId, cfg.baseUrl) + val banner = Banner( + "JOEX", + BuildInfo.version, + BuildInfo.gitHeadCommit, + cfg.jdbc.url, + Option(System.getProperty("config.file")), + cfg.appId, + cfg.baseUrl + ) logger.info(s"\n${banner.render("***>")}") JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 98cd11fc..916974d1 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -16,56 +16,76 @@ object CreateItem { def apply[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = findExisting[F].flatMap { case Some(ri) => Task.pure(ri) - case None => createNew[F] + case None => createNew[F] } def createNew[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = Task { ctx => val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet - def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files). - flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))). - collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }). - zipWithIndex. - evalMap({ case (f, index) => - Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)) - }). - compile.toVector + def fileMetas(itemId: Ident, now: Timestamp) = + Stream + .emits(ctx.args.files) + .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))) + .collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => f }) + .zipWithIndex + .evalMap({ + case (f, index) => + Ident + .randomId[F] + .map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)) + }) + .compile + .toVector - val item = RItem.newItem[F](ctx.args.meta.collective - , ctx.args.makeSubject - , ctx.args.meta.sourceAbbrev - , ctx.args.meta.direction.getOrElse(Direction.Incoming) - , ItemState.Premature) + val item = RItem.newItem[F]( + ctx.args.meta.collective, + ctx.args.makeSubject, + ctx.args.meta.sourceAbbrev, + ctx.args.meta.direction.getOrElse(Direction.Incoming), + ItemState.Premature + ) for { - _ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)") + _ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)") time <- Duration.stopTime[F] - it <- item - n <- ctx.store.transact(RItem.insert(it)) - _ <- if (n != 1) storeItemError[F](ctx) else ().pure[F] - fm <- fileMetas(it.id, it.created) - k <- fm.traverse(a => ctx.store.transact(RAttachment.insert(a))) - _ <- logDifferences(ctx, fm, k.sum) - dur <- time - _ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}") + it <- item + n <- ctx.store.transact(RItem.insert(it)) + _ <- if (n != 1) storeItemError[F](ctx) else ().pure[F] + fm <- fileMetas(it.id, it.created) + k <- fm.traverse(a => ctx.store.transact(RAttachment.insert(a))) + _ <- logDifferences(ctx, fm, k.sum) + dur <- time + _ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}") } yield ItemData(it, fm, Vector.empty, Vector.empty) } def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = Task { ctx => for { - cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) - _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F] - ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) - _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F] - rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))) - } yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)) + cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) + _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") + else ().pure[F] + ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) + _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") + else ().pure[F] + rms <- cand.headOption.traverse(ri => + ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid)) + ) + } yield cand.headOption.map(ri => + ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty) + ) } - private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] = + private def logDifferences[F[_]: Sync]( + ctx: Context[F, ProcessItemArgs], + saved: Vector[RAttachment], + saveCount: Int + ): F[Unit] = if (ctx.args.files.size != saved.size) { - ctx.logger.warn(s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount") + ctx.logger.warn( + s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount" + ) } else { ().pure[F] } diff --git a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala index 5fc47f18..c8b57365 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala @@ -19,45 +19,65 @@ object FindProposal { def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = Task { ctx => - val rmas = data.metas.map(rm => - rm.copy(nerlabels = removeDuplicates(rm.nerlabels))) + val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels))) ctx.logger.info("Starting find-proposal") *> - rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))). - flatMap(rmv => rmv.traverse(rm => - ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *> - ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))). - map(_ => data.copy(metas = rmv))) + rmas + .traverse(rm => + processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml)) + ) + .flatMap(rmv => + rmv + .traverse(rm => + ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *> + ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals)) + ) + .map(_ => data.copy(metas = rmv)) + ) } - def processAttachment[F[_]: Sync]( rm: RAttachmentMeta - , rd: Vector[NerDateLabel] - , ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { + def processAttachment[F[_]: Sync]( + rm: RAttachmentMeta, + rd: Vector[NerDateLabel], + ctx: Context[F, ProcessItemArgs] + ): F[MetaProposalList] = { val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx)) - List(finder.find(rm.nerlabels), makeDateProposal(rd)). - traverse(identity).map(MetaProposalList.flatten) + List(finder.find(rm.nerlabels), makeDateProposal(rd)) + .traverse(identity) + .map(MetaProposalList.flatten) } - def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = { + def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = Timestamp.current[F].map { now => - val latestFirst = dates.sortWith(_.date isAfter _.date) - val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate + val latestFirst = dates.sortWith((l1, l2) => l1.date.isAfter(l2.date)) + val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate)) - val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate, - after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) - val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate, - before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) + val dueDates = MetaProposalList.fromSeq1( + MetaProposalType.DueDate, + after.map(ndl => + Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)) + ) + ) + val itemDates = MetaProposalList.fromSeq1( + MetaProposalType.DocDate, + before.map(ndl => + Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)) + ) + ) MetaProposalList.flatten(Seq(dueDates, itemDates)) } - } def removeDuplicates(labels: List[NerLabel]): List[NerLabel] = - labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) => - if (seen.contains(el.tag.name+el.label.toLowerCase)) (seen, result) - else (seen + (el.tag.name + el.label.toLowerCase), el :: result) - }._2.sortBy(_.startPosition) + labels + .foldLeft((Set.empty[String], List.empty[NerLabel])) { + case ((seen, result), el) => + if (seen.contains(el.tag.name + el.label.toLowerCase)) (seen, result) + else (seen + (el.tag.name + el.label.toLowerCase), el :: result) + } + ._2 + .sortBy(_.startPosition) trait Finder[F[_]] { self => def find(labels: Seq[NerLabel]): F[MetaProposalList] @@ -80,12 +100,14 @@ object FindProposal { else f.map(ml1 => ml0.fillEmptyFrom(ml1)) }) - def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*) - (implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] = - flatMap(res0 => { + def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)( + implicit F: FlatMap[F], + F2: Applicative[F] + ): Finder[F] = + flatMap { res0 => if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0) else f.map(res1 => res0.fillEmptyFrom(res1)) - }) + } } object Finder { @@ -102,7 +124,11 @@ object FindProposal { labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten) } - private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { + private def search[F[_]: Sync]( + nt: NerLabel, + exact: Boolean, + ctx: Context[F, ProcessItemArgs] + ): F[MetaProposalList] = { val value = if (exact) normalizeSearchValue(nt.label) else s"%${normalizeSearchValue(nt.label)}%" @@ -110,70 +136,84 @@ object FindProposal { if (exact) 2 else 5 if (value.length < minLength) { - ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty) - } else nt.tag match { - case NerTag.Organization => - ctx.logger.debug(s"Looking for organizations: $value") *> - ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)). - map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) + ctx.logger + .debug(s"Skipping too small value '$value' (original '${nt.label}').") + .map(_ => MetaProposalList.empty) + } else + nt.tag match { + case NerTag.Organization => + ctx.logger.debug(s"Looking for organizations: $value") *> + ctx.store + .transact(ROrganization.findLike(ctx.args.meta.collective, value)) + .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) - case NerTag.Person => - val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)). - map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) - val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)). - map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) - ctx.logger.debug(s"Looking for persons: $value") *> (for { - ml0 <- s1 - ml1 <- s2 - } yield ml0 |+| ml1) + case NerTag.Person => + val s1 = ctx.store + .transact(RPerson.findLike(ctx.args.meta.collective, value, true)) + .map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) + val s2 = ctx.store + .transact(RPerson.findLike(ctx.args.meta.collective, value, false)) + .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + ctx.logger.debug(s"Looking for persons: $value") *> (for { + ml0 <- s1 + ml1 <- s2 + } yield ml0 |+| ml1) - case NerTag.Location => - ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'."). - map(_ => MetaProposalList.empty) + case NerTag.Location => + ctx.logger + .debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'.") + .map(_ => MetaProposalList.empty) - case NerTag.Misc => - ctx.logger.debug(s"Looking for equipments: $value") *> - ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)). - map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) + case NerTag.Misc => + ctx.logger.debug(s"Looking for equipments: $value") *> + ctx.store + .transact(REquipment.findLike(ctx.args.meta.collective, value)) + .map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) - case NerTag.Email => - searchContact(nt, ContactKind.Email, value, ctx) + case NerTag.Email => + searchContact(nt, ContactKind.Email, value, ctx) - case NerTag.Website => - if (!exact) { - val searchString = Domain.domainFromUri(nt.label.toLowerCase). - toOption. - map(_.toPrimaryDomain.asString). - map(s => s"%$s%"). - getOrElse(value) - searchContact(nt, ContactKind.Website, searchString, ctx) - } else { - searchContact(nt, ContactKind.Website, value, ctx) - } + case NerTag.Website => + if (!exact) { + val searchString = Domain + .domainFromUri(nt.label.toLowerCase) + .toOption + .map(_.toPrimaryDomain.asString) + .map(s => s"%$s%") + .getOrElse(value) + searchContact(nt, ContactKind.Website, searchString, ctx) + } else { + searchContact(nt, ContactKind.Website, value, ctx) + } - case NerTag.Date => - // There is no database search required for this tag - MetaProposalList.empty.pure[F] - } + case NerTag.Date => + // There is no database search required for this tag + MetaProposalList.empty.pure[F] + } } - private def searchContact[F[_]: Sync]( nt: NerLabel - , kind: ContactKind - , value: String - , ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { - val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)). - map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) - val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)). - map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) - val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)). - map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + private def searchContact[F[_]: Sync]( + nt: NerLabel, + kind: ContactKind, + value: String, + ctx: Context[F, ProcessItemArgs] + ): F[MetaProposalList] = { + val orgs = ctx.store + .transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)) + .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) + val corrP = ctx.store + .transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)) + .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + val concP = ctx.store + .transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)) + .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) ctx.logger.debug(s"Looking with $kind: $value") *> List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten) } // The backslash *must* be stripped from search strings. - private [this] val invalidSearch = + private[this] val invalidSearch = "…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet private def normalizeSearchValue(str: String): String = diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index 3092fcdb..a6f751f7 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -4,10 +4,12 @@ import docspell.common.{Ident, NerDateLabel, NerLabel} import docspell.joex.process.ItemData.AttachmentDates import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} -case class ItemData( item: RItem - , attachments: Vector[RAttachment] - , metas: Vector[RAttachmentMeta] - , dateLabels: Vector[AttachmentDates]) { +case class ItemData( + item: RItem, + attachments: Vector[RAttachment], + metas: Vector[RAttachmentMeta], + dateLabels: Vector[AttachmentDates] +) { def findMeta(attachId: Ident): Option[RAttachmentMeta] = metas.find(_.id == attachId) @@ -16,7 +18,6 @@ case class ItemData( item: RItem dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty) } - object ItemData { case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) { @@ -24,4 +25,4 @@ object ItemData { dates.map(dl => dl.label.copy(label = dl.date.toString)) } -} \ No newline at end of file +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 67941365..0d7dda6d 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -10,14 +10,13 @@ import docspell.text.ocr.{Config => OcrConfig} object ItemHandler { def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = - logWarn("Now cancelling. Deleting potentially created data."). - flatMap(_ => deleteByFileIds) + logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ => deleteByFileIds) def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] = - CreateItem[F]. - flatMap(itemStateTask(ItemState.Processing)). - flatMap(safeProcess[F](cfg)). - map(_ => ()) + CreateItem[F] + .flatMap(itemStateTask(ItemState.Processing)) + .flatMap(safeProcess[F](cfg)) + .map(_ => ()) def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] = Task { ctx => @@ -26,26 +25,25 @@ object ItemHandler { def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] = for { - current <- ctx.store.transact(RJob.getRetries(ctx.jobId)) - last = ctx.config.retries == current.getOrElse(0) + current <- ctx.store.transact(RJob.getRetries(ctx.jobId)) + last = ctx.config.retries == current.getOrElse(0) } yield last - - def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def safeProcess[F[_]: Sync: ContextShift]( + cfg: OcrConfig + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = Task(isLastRetry[F, ProcessItemArgs] _).flatMap { case true => - ProcessItem[F](cfg)(data). - attempt.flatMap({ + ProcessItem[F](cfg)(data).attempt.flatMap({ case Right(d) => Task.pure(d) case Left(ex) => - logWarn[F]("Processing failed on last retry. Creating item but without proposals."). - flatMap(_ => itemStateTask(ItemState.Created)(data)). - andThen(_ => Sync[F].raiseError(ex)) + logWarn[F]("Processing failed on last retry. Creating item but without proposals.") + .flatMap(_ => itemStateTask(ItemState.Created)(data)) + .andThen(_ => Sync[F].raiseError(ex)) }) case false => - ProcessItem[F](cfg)(data). - flatMap(itemStateTask(ItemState.Created)) + ProcessItem[F](cfg)(data).flatMap(itemStateTask(ItemState.Created)) } def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala index 81d09236..64a693d2 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -13,28 +13,40 @@ object LinkProposal { val proposals = MetaProposalList.flatten(data.metas.map(_.proposals)) ctx.logger.info(s"Starting linking proposals") *> - MetaProposalType.all. - traverse(applyValue(data, proposals, ctx)). - map(result => ctx.logger.info(s"Results from proposal processing: $result")). - map(_ => data) + MetaProposalType.all + .traverse(applyValue(data, proposals, ctx)) + .map(result => ctx.logger.info(s"Results from proposal processing: $result")) + .map(_ => data) } - def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = { + def applyValue[F[_]: Sync]( + data: ItemData, + proposalList: MetaProposalList, + ctx: Context[F, ProcessItemArgs] + )(mpt: MetaProposalType): F[Result] = proposalList.find(mpt) match { case None => Result.noneFound(mpt).pure[F] case Some(a) if a.isSingleValue => ctx.logger.info(s"Found one candidate for ${a.proposalType}") *> - setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). - map(_ => Result.single(mpt)) + setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ => + Result.single(mpt) + ) case Some(a) => - ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *> - setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). - map(_ => Result.multiple(mpt)) + ctx.logger.info( + s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first." + ) *> + setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ => + Result.multiple(mpt) + ) } - } - def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] = + def setItemMeta[F[_]: Sync]( + itemId: Ident, + ctx: Context[F, ProcessItemArgs], + mpt: MetaProposalType, + value: Ident + ): F[Int] = mpt match { case MetaProposalType.CorrOrg => ctx.logger.debug(s"Updating item organization with: ${value.id}") *> @@ -54,18 +66,17 @@ object LinkProposal { ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0) } - sealed trait Result { def proposalType: MetaProposalType } object Result { - case class NoneFound(proposalType: MetaProposalType) extends Result - case class SingleResult(proposalType: MetaProposalType) extends Result + case class NoneFound(proposalType: MetaProposalType) extends Result + case class SingleResult(proposalType: MetaProposalType) extends Result case class MultipleResult(proposalType: MetaProposalType) extends Result def noneFound(proposalType: MetaProposalType): Result = NoneFound(proposalType) - def single(proposalType: MetaProposalType): Result = SingleResult(proposalType) - def multiple(proposalType: MetaProposalType): Result = MultipleResult(proposalType) + def single(proposalType: MetaProposalType): Result = SingleResult(proposalType) + def multiple(proposalType: MetaProposalType): Result = MultipleResult(proposalType) } } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index d74a7d77..88d16892 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -7,13 +7,15 @@ import docspell.text.ocr.{Config => OcrConfig} object ProcessItem { - def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] = - TextExtraction(cfg, item). - flatMap(Task.setProgress(25)). - flatMap(TextAnalysis[F]). - flatMap(Task.setProgress(50)). - flatMap(FindProposal[F]). - flatMap(Task.setProgress(75)). - flatMap(LinkProposal[F]). - flatMap(Task.setProgress(99)) + def apply[F[_]: Sync: ContextShift]( + cfg: OcrConfig + )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = + TextExtraction(cfg, item) + .flatMap(Task.setProgress(25)) + .flatMap(TextAnalysis[F]) + .flatMap(Task.setProgress(50)) + .flatMap(FindProposal[F]) + .flatMap(Task.setProgress(75)) + .flatMap(LinkProposal[F]) + .flatMap(Task.setProgress(99)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala b/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala index d97709f9..1c69e2db 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala @@ -8,7 +8,7 @@ import docspell.joex.scheduler.Task import org.log4s._ object TestTasks { - private [this] val logger = getLogger + private[this] val logger = getLogger def success[F[_]]: Task[F, ProcessItemArgs, Unit] = Task { ctx => @@ -17,23 +17,23 @@ object TestTasks { def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = Task { ctx => - ctx.logger.info(s"Failing the task run :(").map(_ => - sys.error("Oh, cannot extract gold from this document") - ) + ctx.logger + .info(s"Failing the task run :(") + .map(_ => sys.error("Oh, cannot extract gold from this document")) } def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = Task { ctx => logger.fwarn(s"${Thread.currentThread()} From executing long running task") >> - ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >> - sleep(2400) >> - ctx.logger.debug("doing things") >> - sleep(2400) >> - ctx.logger.debug("doing more things") >> - sleep(2400) >> - ctx.logger.info("doing more things") + ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >> + sleep(2400) >> + ctx.logger.debug("doing things") >> + sleep(2400) >> + ctx.logger.debug("doing more things") >> + sleep(2400) >> + ctx.logger.info("doing more things") } - private def sleep[F[_]:Sync](ms: Long): F[Unit] = + private def sleep[F[_]: Sync](ms: Long): F[Unit] = Sync[F].delay(Thread.sleep(ms)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala index 64bafbcd..7e4cc13b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala @@ -15,35 +15,42 @@ object TextAnalysis { def apply[F[_]: Sync](item: ItemData): Task[F, ProcessItemArgs, ItemData] = Task { ctx => for { - _ <- ctx.logger.info("Starting text analysis") - s <- Duration.stopTime[F] - t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) - _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") - _ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))) - e <- s - _ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") - v = t.toVector + _ <- ctx.logger.info("Starting text analysis") + s <- Duration.stopTime[F] + t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) + _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") + _ <- t.traverse(m => + ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)) + ) + e <- s + _ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") + v = t.toVector } yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2)) } - def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = + def annotateAttachment[F[_]: Sync]( + lang: Language + )(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = for { list0 <- stanfordNer[F](lang, rm) list1 <- contactNer[F](rm) dates <- dateNer[F](rm, lang) } yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates) - def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { - rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty) - } + def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = + Sync[F].delay { + rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty) + } def contactNer[F[_]: Sync](rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { rm.content.map(Contact.annotate).getOrElse(Vector.empty) } def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay { - AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)) + AttachmentDates( + rm, + rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty) + ) } - } diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index 9d7bb565..157fdfee 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -11,10 +11,13 @@ import docspell.text.ocr.{TextExtract, Config => OcrConfig} object TextExtraction { - def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] = + def apply[F[_]: Sync: ContextShift]( + cfg: OcrConfig, + item: ItemData + ): Task[F, ProcessItemArgs, ItemData] = Task { ctx => for { - _ <- ctx.logger.info("Starting text extraction") + _ <- ctx.logger.info("Starting text extraction") start <- Duration.stopTime[F] txt <- item.attachments.traverse(extractTextToMeta(ctx, cfg, ctx.args.meta.language)) _ <- ctx.logger.debug("Storing extracted texts") @@ -24,22 +27,33 @@ object TextExtraction { } yield item.copy(metas = txt) } - def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] = + def extractTextToMeta[F[_]: Sync: ContextShift]( + ctx: Context[F, _], + cfg: OcrConfig, + lang: Language + )(ra: RAttachment): F[RAttachmentMeta] = for { - _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}") - dst <- Duration.stopTime[F] - txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra) - meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty)) - est <- dst - _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}") + _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}") + dst <- Duration.stopTime[F] + txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra) + meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty)) + est <- dst + _ <- ctx.logger.debug( + s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}" + ) } yield meta - def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = { - val data = store.bitpeace.get(ra.fileId.id). - unNoneTerminate. - through(store.bitpeace.fetchData2(RangeDef.all)) + def extractText[F[_]: Sync: ContextShift]( + ocrConfig: OcrConfig, + lang: Language, + store: Store[F], + blocker: Blocker + )(ra: RAttachment): F[Option[String]] = { + val data = store.bitpeace + .get(ra.fileId.id) + .unNoneTerminate + .through(store.bitpeace.fetchData2(RangeDef.all)) - TextExtract.extract(data, blocker, lang.iso3, ocrConfig). - compile.last + TextExtract.extract(data, blocker, lang.iso3, ocrConfig).compile.last } } diff --git a/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala index ec91e40c..a27ea544 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala @@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl object InfoRoutes { def apply[F[_]: Sync](): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { case GET -> (Root / "version") => - Ok(VersionInfo(BuildInfo.version - , BuildInfo.builtAtMillis - , BuildInfo.builtAtString - , BuildInfo.gitHeadCommit.getOrElse("") - , BuildInfo.gitDescribedVersion.getOrElse(""))) + Ok( + VersionInfo( + BuildInfo.version, + BuildInfo.builtAtMillis, + BuildInfo.builtAtString, + BuildInfo.gitHeadCommit.getOrElse(""), + BuildInfo.gitDescribedVersion.getOrElse("") + ) + ) } } } diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala index aa00f910..341be185 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -13,7 +13,7 @@ import org.http4s.dsl.Http4sDsl object JoexRoutes { def apply[F[_]: ConcurrentEffect: Timer](app: JoexApp[F]): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { case POST -> Root / "notify" => @@ -24,14 +24,16 @@ object JoexRoutes { case GET -> Root / "running" => for { - jobs <- app.scheduler.getRunning - jj = jobs.map(mkJob) - resp <- Ok(JobList(jj.toList)) + jobs <- app.scheduler.getRunning + jj = jobs.map(mkJob) + resp <- Ok(JobList(jj.toList)) } yield resp case POST -> Root / "shutdownAndExit" => for { - _ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown) + _ <- ConcurrentEffect[F].start( + Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown + ) resp <- Ok(BasicResult(true, "Shutdown initiated.")) } yield resp @@ -39,20 +41,28 @@ object JoexRoutes { for { optJob <- app.scheduler.getRunning.map(_.find(_.id == id)) optLog <- optJob.traverse(j => app.findLogs(j.id)) - jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) + jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found"))) } yield resp case POST -> Root / "job" / Ident(id) / "cancel" => for { - flag <- app.scheduler.requestCancel(id) + flag <- app.scheduler.requestCancel(id) resp <- Ok(BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found")) } yield resp } } def mkJob(j: RJob): Job = - Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch)) + Job( + j.id, + j.subject, + j.submitted, + j.priority, + j.retries, + j.progress, + j.started.getOrElse(Timestamp.Epoch) + ) def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog = JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList) diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala index 9f4188fa..7ce0b04a 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala @@ -30,40 +30,45 @@ trait Context[F[_], A] { self => } object Context { - private [this] val log = getLogger + private[this] val log = getLogger - def create[F[_]: Functor, A]( job: RJob - , arg: A - , config: SchedulerConfig - , log: Logger[F] - , store: Store[F] - , blocker: Blocker): Context[F, A] = + def create[F[_]: Functor, A]( + job: RJob, + arg: A, + config: SchedulerConfig, + log: Logger[F], + store: Store[F], + blocker: Blocker + ): Context[F, A] = new ContextImpl(arg, log, store, blocker, config, job.id) - def apply[F[_]: Concurrent, A]( job: RJob - , arg: A - , config: SchedulerConfig - , logSink: LogSink[F] - , blocker: Blocker - , store: Store[F]): F[Context[F, A]] = + def apply[F[_]: Concurrent, A]( + job: RJob, + arg: A, + config: SchedulerConfig, + logSink: LogSink[F], + blocker: Blocker, + store: Store[F] + ): F[Context[F, A]] = for { _ <- log.ftrace("Creating logger for task run") logger <- Logger(job.id, job.info, config.logBufferSize, logSink) _ <- log.ftrace("Logger created, instantiating context") - ctx = create[F, A](job, arg, config, logger, store, blocker) + ctx = create[F, A](job, arg, config, logger, store, blocker) } yield ctx - private final class ContextImpl[F[_]: Functor, A]( val args: A - , val logger: Logger[F] - , val store: Store[F] - , val blocker: Blocker - , val config: SchedulerConfig - , val jobId: Ident) - extends Context[F,A] { + final private class ContextImpl[F[_]: Functor, A]( + val args: A, + val logger: Logger[F], + val store: Store[F], + val blocker: Blocker, + val config: SchedulerConfig, + val jobId: Ident + ) extends Context[F, A] { - def setProgress(percent: Int): F[Unit] = { - val pval = math.min(100, math.max(0, percent)) - store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) - } + def setProgress(percent: Int): F[Unit] = { + val pval = math.min(100, math.max(0, percent)) + store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) + } } -} \ No newline at end of file +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala index 3c7771a2..034d4844 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala @@ -11,14 +11,13 @@ import docspell.common.Priority */ case class CountingScheme(high: Int, low: Int, counter: Int = 0) { - def nextPriority: (CountingScheme, Priority) = { + def nextPriority: (CountingScheme, Priority) = if (counter <= 0) (increment, Priority.High) else { val rest = counter % (high + low) if (rest < high) (increment, Priority.High) else (increment, Priority.Low) } - } def increment: CountingScheme = copy(counter = counter + 1) @@ -32,8 +31,7 @@ object CountingScheme { def readString(str: String): Either[String, CountingScheme] = str.split(',') match { case Array(h, l) => - Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)). - left.map(_.getMessage) + Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage) case _ => Left(s"Invalid counting scheme: $str") } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala index 43377d8e..fc69ed1f 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala @@ -20,13 +20,16 @@ case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Tas object JobTask { - def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit]) - (implicit D: Decoder[A]): JobTask[F] = { + def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit])( + implicit D: Decoder[A] + ): JobTask[F] = { val convert: String => F[A] = - str => str.parseJsonAs[A] match { - case Right(a) => a.pure[F] - case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) - } + str => + str.parseJsonAs[A] match { + case Right(a) => a.pure[F] + case Left(ex) => + Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) + } JobTask(name, task.contramap(convert), onCancel.contramap(convert)) } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala index be3a5ff2..f506fb1b 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala @@ -4,12 +4,14 @@ import cats.implicits._ import docspell.common._ import cats.effect.Sync -case class LogEvent( jobId: Ident - , jobInfo: String - , time: Timestamp - , level: LogLevel - , msg: String - , ex: Option[Throwable] = None) { +case class LogEvent( + jobId: Ident, + jobInfo: String, + time: Timestamp, + level: LogLevel, + msg: String, + ex: Option[Throwable] = None +) { def logLine: String = s">>> ${time.asString} $level $jobInfo: $msg" @@ -21,5 +23,4 @@ object LogEvent { def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] = Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) - } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala index b81aa818..b26dd22b 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala @@ -44,12 +44,22 @@ object LogSink { LogSink(_.evalMap(e => logInternal(e))) def db[F[_]: Sync](store: Store[F]): LogSink[F] = - LogSink(_.evalMap(ev => for { - id <- Ident.randomId[F] - joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse("")) - _ <- logInternal(ev) - _ <- store.transact(RJobLog.insert(joblog)) - } yield ())) + LogSink( + _.evalMap(ev => + for { + id <- Ident.randomId[F] + joblog = RJobLog( + id, + ev.jobId, + ev.level, + ev.time, + ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("") + ) + _ <- logInternal(ev) + _ <- store.transact(RJobLog.insert(joblog)) + } yield () + ) + ) def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = { val s: Stream[F, Pipe[F, LogEvent, Unit]] = diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala index 4347255a..353c4182 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala @@ -33,17 +33,25 @@ object Logger { LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1) def error(ex: Throwable)(msg: => String): F[Unit] = - LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1) + LogEvent + .create[F](jobId, jobInfo, LogLevel.Error, msg) + .map(le => le.copy(ex = Some(ex))) + .flatMap(q.enqueue1) def error(msg: => String): F[Unit] = LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1) } - def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] = + def apply[F[_]: Concurrent]( + jobId: Ident, + jobInfo: String, + bufferSize: Int, + sink: LogSink[F] + ): F[Logger[F]] = for { - q <- Queue.circularBuffer[F, LogEvent](bufferSize) - log = create(jobId, jobInfo, q) - _ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain) + q <- Queue.circularBuffer[F, LogEvent](bufferSize) + log = create(jobId, jobInfo, q) + _ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain) } yield log -} \ No newline at end of file +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala index 7e7a9ef5..d1faee33 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala @@ -7,13 +7,14 @@ import docspell.store.Store import docspell.store.queue.JobQueue import fs2.concurrent.SignallingRef -case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( - config: SchedulerConfig - , tasks: JobTaskRegistry[F] - , store: Store[F] - , blocker: Blocker - , queue: Resource[F, JobQueue[F]] - , logSink: LogSink[F]) { +case class SchedulerBuilder[F[_]: ConcurrentEffect: ContextShift]( + config: SchedulerConfig, + tasks: JobTaskRegistry[F], + store: Store[F], + blocker: Blocker, + queue: Resource[F, JobQueue[F]], + logSink: LogSink[F] +) { def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = copy(config = cfg) @@ -33,7 +34,6 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = copy(logSink = sink) - def serve: Resource[F, Scheduler[F]] = resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch)) @@ -45,22 +45,25 @@ case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( perms <- Resource.liftF(Semaphore(config.poolSize.toLong)) } yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms) - scheduler.evalTap(_.init). - map(s => s: Scheduler[F]) + scheduler.evalTap(_.init).map(s => s: Scheduler[F]) } } object SchedulerBuilder { - def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig - , blocker: Blocker - , store: Store[F]): SchedulerBuilder[F] = - new SchedulerBuilder[F](config - , JobTaskRegistry.empty[F] - , store - , blocker - , JobQueue(store) - , LogSink.db[F](store)) + def apply[F[_]: ConcurrentEffect: ContextShift]( + config: SchedulerConfig, + blocker: Blocker, + store: Store[F] + ): SchedulerBuilder[F] = + new SchedulerBuilder[F]( + config, + JobTaskRegistry.empty[F], + store, + blocker, + JobQueue(store), + LogSink.db[F](store) + ) } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala index 2aec4240..6f638dbe 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala @@ -2,24 +2,26 @@ package docspell.joex.scheduler import docspell.common._ -case class SchedulerConfig( name: Ident - , poolSize: Int - , countingScheme: CountingScheme - , retries: Int - , retryDelay: Duration - , logBufferSize: Int - , wakeupPeriod: Duration - ) +case class SchedulerConfig( + name: Ident, + poolSize: Int, + countingScheme: CountingScheme, + retries: Int, + retryDelay: Duration, + logBufferSize: Int, + wakeupPeriod: Duration +) object SchedulerConfig { val default = SchedulerConfig( - name = Ident.unsafe("default-scheduler") - , poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) - , countingScheme = CountingScheme(2, 1) - , retries = 5 - , retryDelay = Duration.seconds(30) - , logBufferSize = 500 - , wakeupPeriod = Duration.minutes(10) + name = Ident.unsafe("default-scheduler"), + poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) + , + countingScheme = CountingScheme(2, 1), + retries = 5, + retryDelay = Duration.seconds(30), + logBufferSize = 500, + wakeupPeriod = Duration.minutes(10) ) } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala index 20546f8e..6ff35a91 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala @@ -14,17 +14,19 @@ import SchedulerImpl._ import docspell.store.Store import docspell.store.queries.QJob -final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig - , blocker: Blocker - , queue: JobQueue[F] - , tasks: JobTaskRegistry[F] - , store: Store[F] - , logSink: LogSink[F] - , state: SignallingRef[F, State[F]] - , waiter: SignallingRef[F, Boolean] - , permits: Semaphore[F]) extends Scheduler[F] { +final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift]( + val config: SchedulerConfig, + blocker: Blocker, + queue: JobQueue[F], + tasks: JobTaskRegistry[F], + store: Store[F], + logSink: LogSink[F], + state: SignallingRef[F, State[F]], + waiter: SignallingRef[F, Boolean], + permits: Semaphore[F] +) extends Scheduler[F] { - private [this] val logger = getLogger + private[this] val logger = getLogger /** * On startup, get all jobs in state running from this scheduler @@ -34,8 +36,13 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch QJob.runningToWaiting(config.name, store) def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] = - ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala). - evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain) + ConcurrentEffect[F].start( + Stream + .awakeEvery[F](config.wakeupPeriod.toScala) + .evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange) + .compile + .drain + ) def getRunning: F[Vector[RJob]] = state.get.flatMap(s => QJob.findAll(s.getRunning, store)) @@ -43,7 +50,7 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch def requestCancel(jobId: Ident): F[Boolean] = state.get.flatMap(_.cancelRequest(jobId) match { case Some(ct) => ct.map(_ => true) - case None => logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) + case None => logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) }) def notifyChange: F[Unit] = @@ -51,59 +58,72 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch def shutdown(cancelAll: Boolean): F[Unit] = { val doCancel = - state.get. - flatMap(_.cancelTokens.values.toList.traverse(identity)). - map(_ => ()) + state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ()) val runShutdown = state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) - val wait = Stream.eval(runShutdown). - evalMap(_ => logger.finfo("Scheduler is shutting down now.")). - flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))). - flatMap(state => { + val wait = Stream + .eval(runShutdown) + .evalMap(_ => logger.finfo("Scheduler is shutting down now.")) + .flatMap(_ => + Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty)) + ) + .flatMap { state => if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running.")) - else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++ - Stream.emit(state) - }) + else + Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++ + Stream.emit(state) + } (wait.drain ++ Stream.emit(())).compile.lastOrError } def start: Stream[F, Nothing] = logger.sinfo("Starting scheduler") ++ - mainLoop + mainLoop def mainLoop: Stream[F, Nothing] = { val body: F[Boolean] = for { - _ <- permits.available.flatMap(a => logger.fdebug(s"Try to acquire permit ($a free)")) - _ <- permits.acquire - _ <- logger.fdebug("New permit acquired") - down <- state.get.map(_.shutdownRequest) - rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] - else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay) - _ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") - _ <- rjob.map(execute).getOrElse(permits.release) + _ <- permits.available.flatMap(a => logger.fdebug(s"Try to acquire permit ($a free)")) + _ <- permits.acquire + _ <- logger.fdebug("New permit acquired") + down <- state.get.map(_.shutdownRequest) + rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] + else + queue.nextJob( + group => state.modify(_.nextPrio(group, config.countingScheme)), + config.name, + config.retryDelay + ) + _ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") + _ <- rjob.map(execute).getOrElse(permits.release) } yield rjob.isDefined - Stream.eval(state.get.map(_.shutdownRequest)). - evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]). - flatMap(if (_) Stream.empty else Stream.eval(body)). - flatMap({ + Stream + .eval(state.get.map(_.shutdownRequest)) + .evalTap( + if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") + else ().pure[F] + ) + .flatMap(if (_) Stream.empty else Stream.eval(body)) + .flatMap({ case true => mainLoop case false => logger.sdebug(s"Waiting for notify") ++ - waiter.discrete.take(2).drain ++ - logger.sdebug(s"Notify signal, going into main loop") ++ - mainLoop + waiter.discrete.take(2).drain ++ + logger.sdebug(s"Notify signal, going into main loop") ++ + mainLoop }) } def execute(job: RJob): F[Unit] = { val task = for { - jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}") + jobtask <- tasks + .find(job.task) + .toRight(s"This executor cannot run tasks with name: ${job.task}") } yield jobtask task match { @@ -122,18 +142,25 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch def onFinish(job: RJob, finalState: JobState): F[Unit] = for { - _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") - _ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)")) - _ <- state.modify(_.removeRunning(job)) - _ <- QJob.setFinalState(job.id, finalState, store) + _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") + _ <- permits.release *> permits.available.flatMap(a => + logger.fdebug(s"Permit released ($a free)") + ) + _ <- state.modify(_.removeRunning(job)) + _ <- QJob.setFinalState(job.id, finalState, store) } yield () def onStart(job: RJob): F[Unit] = QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck - def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = { - task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)). - mapF(_.attempt.flatMap({ + def wrapTask( + job: RJob, + task: Task[F, String, Unit], + ctx: Context[F, String] + ): Task[F, String, Unit] = + task + .mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)) + .mapF(_.attempt.flatMap({ case Right(()) => logger.info(s"Job execution successful: ${job.info}") ctx.logger.info("Job execution successful") *> @@ -148,16 +175,18 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch QJob.exceedsRetries(job.id, config.retries, store).flatMap { case true => logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded."). - map(_ => JobState.Failed: JobState) + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + .map(_ => JobState.Failed: JobState) case false => logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") - ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later."). - map(_ => JobState.Stuck: JobState) + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") + .map(_ => JobState.Stuck: JobState) } } - })). - mapF(_.attempt.flatMap { + })) + .mapF(_.attempt.flatMap { case Right(jstate) => onFinish(job, jstate) case Left(ex) => @@ -165,14 +194,14 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch // we don't know the real outcome here… // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways onFinish(job, JobState.Stuck) - }) - } + }) def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = { val bfa = blocker.blockOn(code) logger.fdebug(s"Forking job ${job.info}") *> - ConcurrentEffect[F].start(bfa). - map(fiber => + ConcurrentEffect[F] + .start(bfa) + .map(fiber => logger.fdebug(s"Cancelling job ${job.info}") *> fiber.cancel *> onCancel.attempt.map({ @@ -184,7 +213,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: Sch state.modify(_.markCancelled(job)) *> onFinish(job, JobState.Cancelled) *> ctx.logger.warn("Job has been cancelled.") *> - logger.fdebug(s"Job ${job.info} has been cancelled.")) + logger.fdebug(s"Job ${job.info} has been cancelled.") + ) } } @@ -193,10 +223,12 @@ object SchedulerImpl { def emptyState[F[_]]: State[F] = State(Map.empty, Set.empty, Map.empty, false) - case class State[F[_]]( counters: Map[Ident, CountingScheme] - , cancelled: Set[Ident] - , cancelTokens: Map[Ident, CancelToken[F]] - , shutdownRequest: Boolean) { + case class State[F[_]]( + counters: Map[Ident, CountingScheme], + cancelled: Set[Ident], + cancelTokens: Map[Ident, CancelToken[F]], + shutdownRequest: Boolean + ) { def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { val (cs, prio) = counters.getOrElse(group, initial).nextPriority diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala index ec041277..8ac59a1b 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala @@ -24,11 +24,11 @@ trait Task[F[_], A, B] { def mapF[C](f: F[B] => F[C]): Task[F, A, C] = Task(Task.toKleisli(this).mapF(f)) - def attempt(implicit F: ApplicativeError[F,Throwable]): Task[F, A, Either[Throwable, B]] = + def attempt(implicit F: ApplicativeError[F, Throwable]): Task[F, A, Either[Throwable, B]] = mapF(_.attempt) - def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { - ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) + def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { ctxc: Context[F, C] => + f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) } } @@ -46,7 +46,6 @@ object Task { def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = c => k.run(c) - def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = Kleisli(t.run) diff --git a/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala b/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala index a84c634c..5d867e87 100644 --- a/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala +++ b/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala @@ -6,8 +6,8 @@ import minitest.SimpleTestSuite object CountingSchemeSpec extends SimpleTestSuite { test("counting") { - val cs = CountingScheme(2,1) - val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2) + val cs = CountingScheme(2, 1) + val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2) val expect = List(Priority.High, Priority.High, Priority.Low) assertEquals(list, expect ++ expect) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/Config.scala b/modules/restserver/src/main/scala/docspell/restserver/Config.scala index 14640661..05b0679b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Config.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Config.scala @@ -7,27 +7,37 @@ import docspell.backend.{Config => BackendConfig} import docspell.common._ import scodec.bits.ByteVector -case class Config(appName: String - , appId: Ident - , baseUrl: LenientUri - , bind: Config.Bind - , backend: BackendConfig - , auth: Login.Config +case class Config( + appName: String, + appId: Ident, + baseUrl: LenientUri, + bind: Config.Bind, + backend: BackendConfig, + auth: Login.Config ) object Config { - val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") - val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") + val postgres = + JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") + val h2 = JdbcConfig( + LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), + "sa", + "" + ) val default: Config = - Config("Docspell" - , Ident.unsafe("restserver1") - , LenientUri.unsafe("http://localhost:7880") - , Config.Bind("localhost", 7880) - , BackendConfig(postgres - , SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)) - , BackendConfig.Files(512 * 1024, List(MimeType.pdf))) - , Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))) + Config( + "Docspell", + Ident.unsafe("restserver1"), + LenientUri.unsafe("http://localhost:7880"), + Config.Bind("localhost", 7880), + BackendConfig( + postgres, + SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)), + BackendConfig.Files(512 * 1024, List(MimeType.pdf)) + ), + Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2)) + ) case class Bind(address: String, port: Int) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala index f0ad397d..d7925ef3 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala @@ -12,7 +12,7 @@ object ConfigFile { ConfigSource.default.at("docspell.server").loadOrThrow[Config] object Implicits { - implicit val signupModeReader: ConfigReader[SignupConfig.Mode] = + implicit val signupModeReader: ConfigReader[SignupConfig.Mode] = ConfigReader[String].emap(reason(SignupConfig.Mode.fromString)) } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/Main.scala b/modules/restserver/src/main/scala/docspell/restserver/Main.scala index 7d8ddb7a..1c9d3d45 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Main.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Main.scala @@ -13,12 +13,14 @@ import org.log4s._ object Main extends IOApp { private[this] val logger = getLogger - val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool( - ThreadFactories.ofName("docspell-restserver-blocking"))) + val blockingEc: ExecutionContext = ExecutionContext.fromExecutor( + Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-restserver-blocking")) + ) val blocker = Blocker.liftExecutionContext(blockingEc) - val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5, - ThreadFactories.ofName("docspell-dbconnect"))) + val connectEC: ExecutionContext = ExecutionContext.fromExecutorService( + Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-dbconnect")) + ) def run(args: List[String]) = { args match { @@ -41,12 +43,15 @@ object Main extends IOApp { } val cfg = ConfigFile.loadConfig - val banner = Banner("REST Server" - , BuildInfo.version - , BuildInfo.gitHeadCommit - , cfg.backend.jdbc.url - , Option(System.getProperty("config.file")) - , cfg.appId, cfg.baseUrl) + val banner = Banner( + "REST Server", + BuildInfo.version, + BuildInfo.gitHeadCommit, + cfg.backend.jdbc.url, + Option(System.getProperty("config.file")), + cfg.appId, + cfg.baseUrl + ) logger.info(s"\n${banner.render("***>")}") RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index aed1b38b..6d559c8c 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -7,7 +7,8 @@ import docspell.common.NodeType import scala.concurrent.ExecutionContext -final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { +final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) + extends RestApp[F] { def init: F[Unit] = backend.node.register(config.appId, NodeType.Restserver, config.baseUrl) @@ -18,11 +19,16 @@ final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[ object RestAppImpl { - def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] = + def create[F[_]: ConcurrentEffect: ContextShift]( + cfg: Config, + connectEC: ExecutionContext, + httpClientEc: ExecutionContext, + blocker: Blocker + ): Resource[F, RestApp[F]] = for { - backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) - app = new RestAppImpl[F](cfg, backend) - appR <- Resource.make(app.init.map(_ => app))(_.shutdown) + backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) + app = new RestAppImpl[F](cfg, backend) + appR <- Resource.make(app.init.map(_ => app))(_.shutdown) } yield appR } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 223822be..93ae891b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -15,54 +15,64 @@ import scala.concurrent.ExecutionContext object RestServer { - def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker) - (implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = { + def stream[F[_]: ConcurrentEffect]( + cfg: Config, + connectEC: ExecutionContext, + httpClientEc: ExecutionContext, + blocker: Blocker + )(implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = { val app = for { - restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker) + restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker) httpApp = Router( - "/api/info" -> routes.InfoRoutes(), + "/api/info" -> routes.InfoRoutes(), "/api/v1/open/" -> openRoutes(cfg, restApp), - "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { - token => securedRoutes(cfg, restApp, token) + "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token => + securedRoutes(cfg, restApp, token) }, "/app/assets" -> WebjarRoutes.appRoutes[F](blocker), - "/app" -> TemplateRoutes[F](blocker, cfg) + "/app" -> TemplateRoutes[F](blocker, cfg) ).orNotFound finalHttpApp = Logger.httpApp(logHeaders = false, logBody = false)(httpApp) } yield finalHttpApp - Stream.resource(app).flatMap(httpApp => - BlazeServerBuilder[F]. - bindHttp(cfg.bind.port, cfg.bind.address). - withHttpApp(httpApp). - withoutBanner. - serve) + Stream + .resource(app) + .flatMap(httpApp => + BlazeServerBuilder[F] + .bindHttp(cfg.bind.port, cfg.bind.address) + .withHttpApp(httpApp) + .withoutBanner + .serve + ) }.drain - - def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] = + def securedRoutes[F[_]: Effect]( + cfg: Config, + restApp: RestApp[F], + token: AuthToken + ): HttpRoutes[F] = Router( - "auth" -> LoginRoutes.session(restApp.backend.login, cfg), - "tag" -> TagRoutes(restApp.backend, token), - "equipment" -> EquipmentRoutes(restApp.backend, token), + "auth" -> LoginRoutes.session(restApp.backend.login, cfg), + "tag" -> TagRoutes(restApp.backend, token), + "equipment" -> EquipmentRoutes(restApp.backend, token), "organization" -> OrganizationRoutes(restApp.backend, token), - "person" -> PersonRoutes(restApp.backend, token), - "source" -> SourceRoutes(restApp.backend, token), - "user" -> UserRoutes(restApp.backend, token), - "collective" -> CollectiveRoutes(restApp.backend, token), - "queue" -> JobQueueRoutes(restApp.backend, token), - "item" -> ItemRoutes(restApp.backend, token), - "attachment" -> AttachmentRoutes(restApp.backend, token), - "upload" -> UploadRoutes.secured(restApp.backend, cfg, token) + "person" -> PersonRoutes(restApp.backend, token), + "source" -> SourceRoutes(restApp.backend, token), + "user" -> UserRoutes(restApp.backend, token), + "collective" -> CollectiveRoutes(restApp.backend, token), + "queue" -> JobQueueRoutes(restApp.backend, token), + "item" -> ItemRoutes(restApp.backend, token), + "attachment" -> AttachmentRoutes(restApp.backend, token), + "upload" -> UploadRoutes.secured(restApp.backend, cfg, token) ) def openRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] = Router( - "auth" -> LoginRoutes.login(restApp.backend.login, cfg), + "auth" -> LoginRoutes.login(restApp.backend.login, cfg), "signup" -> RegisterRoutes(restApp.backend, cfg), "upload" -> UploadRoutes.open(restApp.backend, cfg) ) diff --git a/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala b/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala index 4634a1e3..ffce1c4e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala @@ -8,13 +8,20 @@ import docspell.restserver.Config case class CookieData(auth: AuthToken) { def accountId: AccountId = auth.account - def asString: String = auth.asString + def asString: String = auth.asString def asCookie(cfg: Config): ResponseCookie = { val domain = cfg.baseUrl.host - val sec = cfg.baseUrl.scheme.exists(_.endsWith("s")) - val path = cfg.baseUrl.path/"api"/"v1"/"sec" - ResponseCookie(CookieData.cookieName, asString, domain = domain, path = Some(path.asString), httpOnly = true, secure = sec) + val sec = cfg.baseUrl.scheme.exists(_.endsWith("s")) + val path = cfg.baseUrl.path / "api" / "v1" / "sec" + ResponseCookie( + CookieData.cookieName, + asString, + domain = domain, + path = Some(path.asString), + httpOnly = true, + secure = sec + ) } } object CookieData { @@ -22,18 +29,21 @@ object CookieData { val headerName = "X-Docspell-Auth" def authenticator[F[_]](r: Request[F]): Either[String, String] = - fromCookie(r) orElse fromHeader(r) + fromCookie(r).orElse(fromHeader(r)) - def fromCookie[F[_]](req: Request[F]): Either[String, String] = { + def fromCookie[F[_]](req: Request[F]): Either[String, String] = for { - header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") - cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie") + header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") + cookie <- header.values.toList + .find(_.name == cookieName) + .toRight("Couldn't find the authcookie") } yield cookie.content - } - def fromHeader[F[_]](req: Request[F]): Either[String, String] = { - req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator") - } + def fromHeader[F[_]](req: Request[F]): Either[String, String] = + req.headers + .get(CaseInsensitiveString(headerName)) + .map(_.value) + .toRight("Couldn't find an authenticator") def deleteCookie(cfg: Config): ResponseCookie = ResponseCookie( diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 0ba71b32..a7e3db2e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -24,31 +24,37 @@ trait Conversions { // insights def mkItemInsights(d: InsightData): ItemInsights = - ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))) + ItemInsights( + d.incoming, + d.outgoing, + d.bytes, + TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2))) + ) // attachment meta def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = - AttachmentMeta(rm.content.getOrElse("") - , rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)) - , mkItemProposals(rm.proposals)) - + AttachmentMeta( + rm.content.getOrElse(""), + rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)), + mkItemProposals(rm.proposals) + ) // item proposal def mkItemProposals(ml: MetaProposalList): ItemProposals = { def get(mpt: MetaProposalType) = - ml.find(mpt). - map(mp => mp.values.toList.map(_.ref).map(mkIdName)). - getOrElse(Nil) + ml.find(mpt).map(mp => mp.values.toList.map(_.ref).map(mkIdName)).getOrElse(Nil) def getDates(mpt: MetaProposalType): List[Timestamp] = - ml.find(mpt). - map(mp => mp.values.toList. - map(cand => cand.ref.id.id). - flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption). - map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))). - map(zdt => Timestamp(zdt.toInstant))). - getOrElse(Nil). - distinct. - take(5) + ml.find(mpt) + .map(mp => + mp.values.toList + .map(cand => cand.ref.id.id) + .flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption) + .map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))) + .map(zdt => Timestamp(zdt.toInstant)) + ) + .getOrElse(Nil) + .distinct + .take(5) ItemProposals( corrOrg = get(MetaProposalType.CorrOrg), @@ -62,23 +68,25 @@ trait Conversions { // item detail def mkItemDetail(data: OItem.ItemData): ItemDetail = - ItemDetail(data.item.id - , data.item.direction - , data.item.name - , data.item.source - , data.item.state - , data.item.created - , data.item.updated - , data.item.itemDate - , data.corrOrg.map(o => IdName(o.oid, o.name)) - , data.corrPerson.map(p => IdName(p.pid, p.name)) - , data.concPerson.map(p => IdName(p.pid, p.name)) - , data.concEquip.map(e => IdName(e.eid, e.name)) - , data.inReplyTo.map(mkIdName) - , data.item.dueDate - , data.item.notes - , data.attachments.map((mkAttachment _).tupled).toList - , data.tags.map(mkTag).toList) + ItemDetail( + data.item.id, + data.item.direction, + data.item.name, + data.item.source, + data.item.state, + data.item.created, + data.item.updated, + data.item.itemDate, + data.corrOrg.map(o => IdName(o.oid, o.name)), + data.corrPerson.map(p => IdName(p.pid, p.name)), + data.concPerson.map(p => IdName(p.pid, p.name)), + data.concEquip.map(e => IdName(e.eid, e.name)), + data.inReplyTo.map(mkIdName), + data.item.dueDate, + data.item.notes, + data.attachments.map((mkAttachment _).tupled).toList, + data.tags.map(mkTag).toList + ) def mkAttachment(ra: RAttachment, m: FileMeta): Attachment = Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString)) @@ -86,20 +94,21 @@ trait Conversions { // item list def mkQuery(m: ItemSearch, coll: Ident): OItem.Query = - OItem.Query(coll - , m.name - , if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed) - , m.direction - , m.corrPerson - , m.corrOrg - , m.concPerson - , m.concEquip - , m.tagsInclude.map(Ident.unsafe) - , m.tagsExclude.map(Ident.unsafe) - , m.dateFrom - , m.dateUntil - , m.dueDateFrom - , m.dueDateUntil + OItem.Query( + coll, + m.name, + if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed), + m.direction, + m.corrPerson, + m.corrOrg, + m.concPerson, + m.concEquip, + m.tagsInclude.map(Ident.unsafe), + m.tagsExclude.map(Ident.unsafe), + m.dateFrom, + m.dateUntil, + m.dueDateFrom, + m.dueDateUntil ) def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = { @@ -113,8 +122,20 @@ trait Conversions { } def mkItemLight(i: OItem.ListItem): ItemLight = - ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName), - i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount) + ItemLight( + i.id, + i.name, + i.state, + i.date, + i.dueDate, + i.source, + i.direction.name.some, + i.corrOrg.map(mkIdName), + i.corrPerson.map(mkIdName), + i.concPerson.map(mkIdName), + i.concEquip.map(mkIdName), + i.fileCount + ) // job def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { @@ -128,46 +149,57 @@ trait Conversions { val t2 = f(j2).getOrElse(Timestamp.Epoch) t1.value.isBefore(t2.value) } - JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started)) - , state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)) - , state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))) + JobQueueState( + state.running.map(mkJobDetail).toList.sortWith(asc(_.started)), + state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)), + state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some)) + ) } def mkJobDetail(jd: OJob.JobDetail): JobDetail = - JobDetail(jd.job.id - , jd.job.subject - , jd.job.submitted - , jd.job.priority - , jd.job.state - , jd.job.retries - , jd.logs.map(mkJobLog).toList - , jd.job.progress - , jd.job.worker - , jd.job.started - , jd.job.finished) + JobDetail( + jd.job.id, + jd.job.subject, + jd.job.submitted, + jd.job.priority, + jd.job.state, + jd.job.retries, + jd.logs.map(mkJobLog).toList, + jd.job.progress, + jd.job.worker, + jd.job.started, + jd.job.finished + ) def mkJobLog(jl: RJobLog): JobLogEvent = JobLogEvent(jl.created, jl.level, jl.message) // upload - def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = { - def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = { - body.through(fs2.text.utf8Decode). - parseJsonAs[ItemUploadMeta]. - map(_.fold(ex => { + def readMultipart[F[_]: Effect]( + mp: Multipart[F], + logger: Logger, + prio: Priority, + validFileTypes: Seq[MimeType] + ): F[UploadData[F]] = { + def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = + body + .through(fs2.text.utf8Decode) + .parseJsonAs[ItemUploadMeta] + .map(_.fold(ex => { logger.error(ex)("Reading upload metadata failed.") throw ex }, identity)) - } - val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")). - map(p => parseMeta(p.body)). - map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))). - getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) + val meta: F[(Boolean, UploadMeta)] = mp.parts + .find(_.name.exists(_.equalsIgnoreCase("meta"))) + .map(p => parseMeta(p.body)) + .map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))) + .getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) - val files = mp.parts. - filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))). - map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)) + val files = mp.parts + .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))) + .map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body) + ) for { metaData <- meta _ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData")) @@ -178,8 +210,14 @@ trait Conversions { // organization and person def mkOrg(v: OOrganization.OrgAndContacts): Organization = { val ro = v.org - Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), - v.contacts.map(mkContact).toList, ro.notes, ro.created) + Organization( + ro.oid, + ro.name, + Address(ro.street, ro.zip, ro.city, ro.country), + v.contacts.map(mkContact).toList, + ro.notes, + ro.created + ) } def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = { @@ -189,7 +227,17 @@ trait Conversions { now <- Timestamp.current[F] oid <- Ident.randomId[F] cont <- contacts(oid) - org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now) + org = ROrganization( + oid, + cid, + v.name, + v.address.street, + v.address.zip, + v.address.city, + v.address.country, + v.notes, + now + ) } yield OOrganization.OrgAndContacts(org, cont) } @@ -197,15 +245,32 @@ trait Conversions { def contacts(oid: Ident) = v.contacts.traverse(c => newContact(c, oid.some, None)) for { - cont <- contacts(v.id) - org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created) + cont <- contacts(v.id) + org = ROrganization( + v.id, + cid, + v.name, + v.address.street, + v.address.zip, + v.address.city, + v.address.country, + v.notes, + v.created + ) } yield OOrganization.OrgAndContacts(org, cont) } def mkPerson(v: OOrganization.PersonAndContacts): Person = { val ro = v.person - Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), - v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created) + Person( + ro.pid, + ro.name, + Address(ro.street, ro.zip, ro.city, ro.country), + v.contacts.map(mkContact).toList, + ro.notes, + ro.concerning, + ro.created + ) } def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = { @@ -215,7 +280,18 @@ trait Conversions { now <- Timestamp.current[F] pid <- Ident.randomId[F] cont <- contacts(pid) - org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now) + org = RPerson( + pid, + cid, + v.name, + v.address.street, + v.address.zip, + v.address.city, + v.address.country, + v.notes, + v.concerning, + now + ) } yield OOrganization.PersonAndContacts(org, cont) } @@ -223,8 +299,19 @@ trait Conversions { def contacts(pid: Ident) = v.contacts.traverse(c => newContact(c, None, pid.some)) for { - cont <- contacts(v.id) - org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created) + cont <- contacts(v.id) + org = RPerson( + v.id, + cid, + v.name, + v.address.street, + v.address.zip, + v.address.city, + v.address.country, + v.notes, + v.concerning, + v.created + ) } yield OOrganization.PersonAndContacts(org, cont) } @@ -233,7 +320,8 @@ trait Conversions { Contact(rc.contactId, rc.value, rc.kind) def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] = - timeId.map { case (id, now) => + timeId.map { + case (id, now) => RContact(id, c.value, c.kind, pid, oid, now) } @@ -242,12 +330,33 @@ trait Conversions { User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created) def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = - timeId.map { case (id, now) => - RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, 0, None, now) + timeId.map { + case (id, now) => + RUser( + id, + u.login, + cid, + u.password.getOrElse(Password.empty), + u.state, + u.email, + 0, + None, + now + ) } def changeUser(u: User, cid: Ident): RUser = - RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created) + RUser( + Ident.unsafe(""), + u.login, + cid, + u.password.getOrElse(Password.empty), + u.state, + u.email, + u.loginCount, + u.lastLogin, + u.created + ) // tags @@ -255,34 +364,36 @@ trait Conversions { Tag(rt.tagId, rt.name, rt.category, rt.created) def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = - timeId.map { case (id, now) => - RTag(id, cid, t.name, t.category, now) + timeId.map { + case (id, now) => + RTag(id, cid, t.name, t.category, now) } def changeTag(t: Tag, cid: Ident): RTag = RTag(t.id, cid, t.name, t.category, t.created) - // sources def mkSource(s: RSource): Source = Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = - timeId.map({ case (id, now) => - RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) + timeId.map({ + case (id, now) => + RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) }) def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource = - RSource(s.id, coll, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) + RSource(s.id, coll, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) // equipment def mkEquipment(re: REquipment): Equipment = Equipment(re.eid, re.name, re.created) def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = - timeId.map({ case (id, now) => - REquipment(id, cid, e.name, now) + timeId.map({ + case (id, now) => + REquipment(id, cid, e.name, now) }) def changeEquipment(e: Equipment, cid: Ident): REquipment = @@ -298,26 +409,28 @@ trait Conversions { def basicResult(cr: JobCancelResult): BasicResult = cr match { case JobCancelResult.JobNotFound => BasicResult(false, "Job not found") - case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor") + case JobCancelResult.CancelRequested => + BasicResult(true, "Cancel was requested at the job executor") case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.") } def basicResult(ar: AddResult, successMsg: String): BasicResult = ar match { - case AddResult.Success => BasicResult(true, successMsg) + case AddResult.Success => BasicResult(true, successMsg) case AddResult.EntityExists(msg) => BasicResult(false, msg) - case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}") + case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}") } def basicResult(ur: OUpload.UploadResult): BasicResult = ur match { - case UploadResult.Success => BasicResult(true, "Files submitted.") - case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.") + case UploadResult.Success => BasicResult(true, "Files submitted.") + case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.") case UploadResult.NoSource => BasicResult(false, "The source id is not valid.") } def basicResult(cr: PassChangeResult): BasicResult = cr match { - case PassChangeResult.Success => BasicResult(true, "Password changed.") + case PassChangeResult.Success => BasicResult(true, "Password changed.") case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.") - case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.") + case PassChangeResult.PasswordMismatch => + BasicResult(false, "The current password is incorrect.") case PassChangeResult.UserNotFound => BasicResult(false, "User not found.") } diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala index 06f26ab1..3a349760 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala @@ -8,28 +8,26 @@ import org.http4s.dsl.Http4sDsl trait ResponseGenerator[F[_]] { self: Http4sDsl[F] => - - implicit final class EitherResponses[A,B](e: Either[A, B]) { - def toResponse(headers: Header*) - (implicit F: Applicative[F] - , w0: EntityEncoder[F, A] - , w1: EntityEncoder[F, B]): F[Response[F]] = + implicit final class EitherResponses[A, B](e: Either[A, B]) { + def toResponse(headers: Header*)( + implicit F: Applicative[F], + w0: EntityEncoder[F, A], + w1: EntityEncoder[F, B] + ): F[Response[F]] = e.fold( - a => UnprocessableEntity(a), - b => Ok(b) - ).map(_.withHeaders(headers: _*)) + a => UnprocessableEntity(a), + b => Ok(b) + ) + .map(_.withHeaders(headers: _*)) } implicit final class OptionResponse[A](o: Option[A]) { - def toResponse(headers: Header*) - (implicit F: Applicative[F] - , w0: EntityEncoder[F, A]): F[Response[F]] = + def toResponse( + headers: Header* + )(implicit F: Applicative[F], w0: EntityEncoder[F, A]): F[Response[F]] = o.map(a => Ok(a)).getOrElse(NotFound()).map(_.withHeaders(headers: _*)) } } -object ResponseGenerator { - - -} +object ResponseGenerator {} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala index 525699f8..2ccfd897 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala @@ -18,17 +18,18 @@ import org.http4s.headers.ETag.EntityTag object AttachmentRoutes { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = { - val mt = MediaType.unsafeParse(data.meta.mimetype.asString) + val mt = MediaType.unsafeParse(data.meta.mimetype.asString) val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length) - val eTag: Header = ETag(data.meta.checksum) - val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse(""))) - Ok(data.data.take(data.meta.length)). - map(r => r.withContentType(`Content-Type`(mt)). - withHeaders(cntLen, eTag, disp)) + val eTag: Header = ETag(data.meta.checksum) + val disp: Header = + `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse(""))) + Ok(data.data.take(data.meta.length)).map(r => + r.withContentType(`Content-Type`(mt)).withHeaders(cntLen, eTag, disp) + ) } HttpRoutes.of { @@ -37,21 +38,24 @@ object AttachmentRoutes { fileData <- backend.item.findAttachment(id, user.account.collective) inm = req.headers.get(`If-None-Match`).flatMap(_.tags) matches = matchETag(fileData, inm) - resp <- if (matches) NotModified() - else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found"))) + resp <- if (matches) NotModified() + else + fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found"))) } yield resp case GET -> Root / Ident(id) / "meta" => for { - rm <- backend.item.findAttachmentMeta(id, user.account.collective) + rm <- backend.item.findAttachmentMeta(id, user.account.collective) md = rm.map(Conversions.mkAttachmentMeta) resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found."))) } yield resp } } - private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]] - , noneMatch: Option[NonEmptyList[EntityTag]]): Boolean = + private def matchETag[F[_]]( + fileData: Option[OItem.AttachmentData[F]], + noneMatch: Option[NonEmptyList[EntityTag]] + ): Boolean = (fileData, noneMatch) match { case (Some(fd), Some(nm)) => fd.meta.checksum == nm.head.tag diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala index e07de9d0..1218cbe1 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala @@ -12,14 +12,17 @@ import org.http4s.server._ object Authenticate { - def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] = + def authenticateRequest[F[_]: Effect]( + auth: String => F[Login.Result] + )(req: Request[F]): F[Login.Result] = CookieData.authenticator(req) match { case Right(str) => auth(str) - case Left(_) => Login.Result.invalidAuth.pure[F] + case Left(_) => Login.Result.invalidAuth.pure[F] } - - def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = { + def of[F[_]: Effect](S: Login[F], cfg: Login.Config)( + pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]] + ): HttpRoutes[F] = { val dsl: Http4sDsl[F] = new Http4sDsl[F] {} import dsl._ @@ -34,7 +37,9 @@ object Authenticate { middleware(AuthedRoutes.of(pf)) } - def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = { + def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)( + f: AuthToken => HttpRoutes[F] + ): HttpRoutes[F] = { val dsl: Http4sDsl[F] = new Http4sDsl[F] {} import dsl._ @@ -49,6 +54,8 @@ object Authenticate { middleware(AuthedRoutes(authReq => f(authReq.context).run(authReq.req))) } - private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] = + private def getUser[F[_]: Effect]( + auth: String => F[Login.Result] + ): Kleisli[F, Request[F], Either[String, AuthToken]] = Kleisli(r => authenticateRequest(auth)(r).map(_.toEither)) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala index 7b36fa37..44a25356 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala @@ -25,25 +25,25 @@ object CollectiveRoutes { resp <- Ok(Conversions.mkItemInsights(ins)) } yield resp - case req@POST -> Root / "settings" => + case req @ POST -> Root / "settings" => for { - settings <- req.as[CollectiveSettings] - res <- backend.collective.updateLanguage(user.account.collective, settings.language) - resp <- Ok(Conversions.basicResult(res, "Language updated.")) + settings <- req.as[CollectiveSettings] + res <- backend.collective.updateLanguage(user.account.collective, settings.language) + resp <- Ok(Conversions.basicResult(res, "Language updated.")) } yield resp case GET -> Root / "settings" => for { - collDb <- backend.collective.find(user.account.collective) - sett = collDb.map(c => CollectiveSettings(c.language)) - resp <- sett.toResponse() + collDb <- backend.collective.find(user.account.collective) + sett = collDb.map(c => CollectiveSettings(c.language)) + resp <- sett.toResponse() } yield resp case GET -> Root => for { - collDb <- backend.collective.find(user.account.collective) - coll = collDb.map(c => Collective(c.id, c.state, c.created)) - resp <- coll.toResponse() + collDb <- backend.collective.find(user.account.collective) + coll = collDb.map(c => Collective(c.id, c.state, c.created)) + resp <- coll.toResponse() } yield resp } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala index 5f276eff..6dfeede7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala @@ -15,7 +15,7 @@ import org.http4s.dsl.Http4sDsl object EquipmentRoutes { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of { @@ -36,12 +36,12 @@ object EquipmentRoutes { case req @ PUT -> Root => for { data <- req.as[Equipment] - equip = changeEquipment(data, user.account.collective) + equip = changeEquipment(data, user.account.collective) res <- backend.equipment.update(equip) resp <- Ok(basicResult(res, "Equipment updated.")) } yield resp - case DELETE -> Root / Ident(id) => + case DELETE -> Root / Ident(id) => for { del <- backend.equipment.delete(id, user.account.collective) resp <- Ok(basicResult(del, "Equipment deleted.")) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala index 978faad3..0ab1de86 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala @@ -10,15 +10,19 @@ import org.http4s.dsl.Http4sDsl object InfoRoutes { def apply[F[_]: Sync](): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { case GET -> (Root / "version") => - Ok(VersionInfo(BuildInfo.version - , BuildInfo.builtAtMillis - , BuildInfo.builtAtString - , BuildInfo.gitHeadCommit.getOrElse("") - , BuildInfo.gitDescribedVersion.getOrElse(""))) + Ok( + VersionInfo( + BuildInfo.version, + BuildInfo.builtAtMillis, + BuildInfo.builtAtString, + BuildInfo.gitHeadCommit.getOrElse(""), + BuildInfo.gitDescribedVersion.getOrElse("") + ) + ) } } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index eb6b91fa..a35a69cc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -18,24 +18,24 @@ object ItemRoutes { private[this] val logger = getLogger def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of { case req @ POST -> Root / "search" => for { - mask <- req.as[ItemSearch] - _ <- logger.ftrace(s"Got search mask: $mask") - query = Conversions.mkQuery(mask, user.account.collective) - _ <- logger.ftrace(s"Running query: $query") - items <- backend.item.findItems(query, 100) - resp <- Ok(Conversions.mkItemList(items)) + mask <- req.as[ItemSearch] + _ <- logger.ftrace(s"Got search mask: $mask") + query = Conversions.mkQuery(mask, user.account.collective) + _ <- logger.ftrace(s"Running query: $query") + items <- backend.item.findItems(query, 100) + resp <- Ok(Conversions.mkItemList(items)) } yield resp case GET -> Root / Ident(id) => for { item <- backend.item.findItem(id, user.account.collective) - result = item.map(Conversions.mkItemDetail) + result = item.map(Conversions.mkItemDetail) resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found."))) } yield resp @@ -51,89 +51,89 @@ object ItemRoutes { resp <- Ok(Conversions.basicResult(res, "Item back to created.")) } yield resp - case req@POST -> Root / Ident(id) / "tags" => + case req @ POST -> Root / Ident(id) / "tags" => for { - tags <- req.as[ReferenceList].map(_.items) - res <- backend.item.setTags(id, tags.map(_.id), user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Tags updated")) + tags <- req.as[ReferenceList].map(_.items) + res <- backend.item.setTags(id, tags.map(_.id), user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Tags updated")) } yield resp - case req@POST -> Root / Ident(id) / "direction" => + case req @ POST -> Root / Ident(id) / "direction" => for { dir <- req.as[DirectionValue] res <- backend.item.setDirection(id, dir.direction, user.account.collective) resp <- Ok(Conversions.basicResult(res, "Direction updated")) } yield resp - case req@POST -> Root / Ident(id) / "corrOrg" => + case req @ POST -> Root / Ident(id) / "corrOrg" => for { - idref <- req.as[OptionalId] - res <- backend.item.setCorrOrg(id, idref.id, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated")) + idref <- req.as[OptionalId] + res <- backend.item.setCorrOrg(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated")) } yield resp - case req@POST -> Root / Ident(id) / "corrPerson" => + case req @ POST -> Root / Ident(id) / "corrPerson" => for { - idref <- req.as[OptionalId] - res <- backend.item.setCorrPerson(id, idref.id, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Correspondent person updated")) + idref <- req.as[OptionalId] + res <- backend.item.setCorrPerson(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Correspondent person updated")) } yield resp - case req@POST -> Root / Ident(id) / "concPerson" => + case req @ POST -> Root / Ident(id) / "concPerson" => for { - idref <- req.as[OptionalId] - res <- backend.item.setConcPerson(id, idref.id, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Concerned person updated")) + idref <- req.as[OptionalId] + res <- backend.item.setConcPerson(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned person updated")) } yield resp - case req@POST -> Root / Ident(id) / "concEquipment" => + case req @ POST -> Root / Ident(id) / "concEquipment" => for { - idref <- req.as[OptionalId] - res <- backend.item.setConcEquip(id, idref.id, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) - } yield resp - - case req@POST -> Root / Ident(id) / "notes" => - for { - text <- req.as[OptionalText] - res <- backend.item.setNotes(id, text.text, user.account.collective) + idref <- req.as[OptionalId] + res <- backend.item.setConcEquip(id, idref.id, user.account.collective) resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) } yield resp - case req@POST -> Root / Ident(id) / "name" => + case req @ POST -> Root / Ident(id) / "notes" => for { - text <- req.as[OptionalText] - res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) + text <- req.as[OptionalText] + res <- backend.item.setNotes(id, text.text, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) } yield resp - case req@POST -> Root / Ident(id) / "duedate" => + case req @ POST -> Root / Ident(id) / "name" => for { - date <- req.as[OptionalDate] - _ <- logger.fdebug(s"Setting item due date to ${date.date}") - res <- backend.item.setItemDueDate(id, date.date, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Item due date updated")) + text <- req.as[OptionalText] + res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) } yield resp - case req@POST -> Root / Ident(id) / "date" => + case req @ POST -> Root / Ident(id) / "duedate" => for { - date <- req.as[OptionalDate] - _ <- logger.fdebug(s"Setting item date to ${date.date}") - res <- backend.item.setItemDate(id, date.date, user.account.collective) - resp <- Ok(Conversions.basicResult(res, "Item date updated")) + date <- req.as[OptionalDate] + _ <- logger.fdebug(s"Setting item due date to ${date.date}") + res <- backend.item.setItemDueDate(id, date.date, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item due date updated")) + } yield resp + + case req @ POST -> Root / Ident(id) / "date" => + for { + date <- req.as[OptionalDate] + _ <- logger.fdebug(s"Setting item date to ${date.date}") + res <- backend.item.setItemDate(id, date.date, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item date updated")) } yield resp case GET -> Root / Ident(id) / "proposals" => for { ml <- backend.item.getProposals(id, user.account.collective) - ip = Conversions.mkItemProposals(ml) + ip = Conversions.mkItemProposals(ml) resp <- Ok(ip) } yield resp case DELETE -> Root / Ident(id) => for { n <- backend.item.delete(id, user.account.collective) - res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.") + res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.") resp <- Ok(res) } yield resp } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala index 4539e90b..d62b35e9 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala @@ -19,15 +19,15 @@ object JobQueueRoutes { HttpRoutes.of { case GET -> Root / "state" => for { - js <- backend.job.queueState(user.account.collective, 200) + js <- backend.job.queueState(user.account.collective, 200) res = Conversions.mkJobQueueState(js) resp <- Ok(res) } yield resp case POST -> Root / Ident(id) / "cancel" => for { - result <- backend.job.cancelJob(id, user.account.collective) - resp <- Ok(Conversions.basicResult(result)) + result <- backend.job.cancelJob(id, user.account.collective) + resp <- Ok(Conversions.basicResult(result)) } yield resp } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala index 23feb354..26e619d1 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala @@ -18,10 +18,10 @@ object LoginRoutes { import dsl._ HttpRoutes.of[F] { - case req@POST -> Root / "login" => + case req @ POST -> Root / "login" => for { - up <- req.as[UserPass] - res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password)) + up <- req.as[UserPass] + res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password)) resp <- makeResponse(dsl, cfg, res, up.account) } yield resp } @@ -33,22 +33,36 @@ object LoginRoutes { HttpRoutes.of[F] { case req @ POST -> Root / "session" => - Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req). - flatMap(res => makeResponse(dsl, cfg, res, "")) + Authenticate + .authenticateRequest(S.loginSession(cfg.auth))(req) + .flatMap(res => makeResponse(dsl, cfg, res, "")) case POST -> Root / "logout" => Ok().map(_.addCookie(CookieData.deleteCookie(cfg))) } } - def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = { + def makeResponse[F[_]: Effect]( + dsl: Http4sDsl[F], + cfg: Config, + res: Login.Result, + account: String + ): F[Response[F]] = { import dsl._ res match { case Login.Result.Ok(token) => for { cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply) - resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)). - map(_.addCookie(cd.asCookie(cfg))) + resp <- Ok( + AuthResult( + token.account.collective.id, + token.account.user.id, + true, + "Login successful", + Some(cd.asString), + cfg.auth.sessionValid.millis + ) + ).map(_.addCookie(cd.asCookie(cfg))) } yield resp case _ => Ok(AuthResult("", account, false, "Login failed.", None, 0L)) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala index ee56422c..dc4fc494 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala @@ -16,15 +16,15 @@ import org.http4s.dsl.Http4sDsl object OrganizationRoutes { def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of { case GET -> Root :? FullQueryParamMatcher(full) => if (full.getOrElse(false)) { for { - data <- backend.organization.findAllOrg(user.account) - resp <- Ok(OrganizationList(data.map(mkOrg).toList)) + data <- backend.organization.findAllOrg(user.account) + resp <- Ok(OrganizationList(data.map(mkOrg).toList)) } yield resp } else { for { @@ -38,7 +38,7 @@ object OrganizationRoutes { data <- req.as[Organization] newOrg <- newOrg(data, user.account.collective) added <- backend.organization.addOrg(newOrg) - resp <- Ok(basicResult(added, "New organization saved.")) + resp <- Ok(basicResult(added, "New organization saved.")) } yield resp case req @ PUT -> Root => @@ -49,10 +49,10 @@ object OrganizationRoutes { resp <- Ok(basicResult(update, "Organization updated.")) } yield resp - case DELETE -> Root / Ident(id) => + case DELETE -> Root / Ident(id) => for { - delOrg <- backend.organization.deleteOrg(id, user.account.collective) - resp <- Ok(basicResult(delOrg, "Organization deleted.")) + delOrg <- backend.organization.deleteOrg(id, user.account.collective) + resp <- Ok(basicResult(delOrg, "Organization deleted.")) } yield resp } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala index 3516c4c5..3fe1aac7 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala @@ -6,9 +6,10 @@ import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher object ParamDecoder { implicit val booleanDecoder: QueryParamDecoder[Boolean] = - QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean") + QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_.equalsIgnoreCase("true")))( + "Boolean" + ) object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full") - } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala index d69ecf31..b7fe174a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala @@ -19,15 +19,15 @@ object PersonRoutes { private[this] val logger = getLogger def apply[F[_]: Effect](backend: BackendApp[F], user: AuthToken): HttpRoutes[F] = { - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of { - case GET -> Root :? FullQueryParamMatcher(full) => + case GET -> Root :? FullQueryParamMatcher(full) => if (full.getOrElse(false)) { for { - data <- backend.organization.findAllPerson(user.account) - resp <- Ok(PersonList(data.map(mkPerson).toList)) + data <- backend.organization.findAllPerson(user.account) + resp <- Ok(PersonList(data.map(mkPerson).toList)) } yield resp } else { for { @@ -41,7 +41,7 @@ object PersonRoutes { data <- req.as[Person] newPer <- newPerson(data, user.account.collective) added <- backend.organization.addPerson(newPer) - resp <- Ok(basicResult(added, "New person saved.")) + resp <- Ok(basicResult(added, "New person saved.")) } yield resp case req @ PUT -> Root => @@ -52,11 +52,11 @@ object PersonRoutes { resp <- Ok(basicResult(update, "Person updated.")) } yield resp - case DELETE -> Root / Ident(id) => + case DELETE -> Root / Ident(id) => for { - _ <- logger.fdebug(s"Deleting person ${id.id}") - delOrg <- backend.organization.deletePerson(id, user.account.collective) - resp <- Ok(basicResult(delOrg, "Person deleted.")) + _ <- logger.fdebug(s"Deleting person ${id.id}") + delOrg <- backend.organization.deletePerson(id, user.account.collective) + resp <- Ok(basicResult(delOrg, "Person deleted.")) } yield resp } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala index 0b425075..29424eb4 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala @@ -24,16 +24,16 @@ object RegisterRoutes { HttpRoutes.of { case req @ POST -> Root / "register" => for { - data <- req.as[Registration] - res <- backend.signup.register(cfg.backend.signup)(convert(data)) - resp <- Ok(convert(res)) + data <- req.as[Registration] + res <- backend.signup.register(cfg.backend.signup)(convert(data)) + resp <- Ok(convert(res)) } yield resp - case req@ POST -> Root / "newinvite" => + case req @ POST -> Root / "newinvite" => for { - data <- req.as[GenInvite] - res <- backend.signup.newInvite(cfg.backend.signup)(data.password) - resp <- Ok(convert(res)) + data <- req.as[GenInvite] + res <- backend.signup.newInvite(cfg.backend.signup)(data.password) + resp <- Ok(convert(res)) } yield resp } } @@ -47,7 +47,6 @@ object RegisterRoutes { InviteResult(false, "Password is invalid.", None) } - def convert(r: SignupResult): BasicResult = r match { case SignupResult.CollectiveExists => BasicResult(false, "A collective with this name already exists.") @@ -62,7 +61,6 @@ object RegisterRoutes { BasicResult(true, "Signup successful") } - def convert(r: Registration): RegisterData = RegisterData(r.collectiveName, r.login, r.password, r.invite) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala index b90640a1..ce7e65f9 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala @@ -22,8 +22,8 @@ object SourceRoutes { HttpRoutes.of { case GET -> Root => for { - all <- backend.source.findAll(user.account) - res <- Ok(SourceList(all.map(mkSource).toList)) + all <- backend.source.findAll(user.account) + res <- Ok(SourceList(all.map(mkSource).toList)) } yield res case req @ POST -> Root => @@ -37,12 +37,12 @@ object SourceRoutes { case req @ PUT -> Root => for { data <- req.as[Source] - src = changeSource(data, user.account.collective) + src = changeSource(data, user.account.collective) updated <- backend.source.update(src) resp <- Ok(basicResult(updated, "Source updated.")) } yield resp - case DELETE -> Root / Ident(id) => + case DELETE -> Root / Ident(id) => for { del <- backend.source.delete(id, user.account.collective) resp <- Ok(basicResult(del, "Source deleted.")) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala index 3efdae72..0fc8579e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala @@ -28,21 +28,21 @@ object TagRoutes { case req @ POST -> Root => for { - data <- req.as[Tag] - tag <- newTag(data, user.account.collective) - res <- backend.tag.add(tag) - resp <- Ok(basicResult(res, "Tag successfully created.")) + data <- req.as[Tag] + tag <- newTag(data, user.account.collective) + res <- backend.tag.add(tag) + resp <- Ok(basicResult(res, "Tag successfully created.")) } yield resp case req @ PUT -> Root => for { - data <- req.as[Tag] - tag = changeTag(data, user.account.collective) - res <- backend.tag.update(tag) - resp <- Ok(basicResult(res, "Tag successfully updated.")) + data <- req.as[Tag] + tag = changeTag(data, user.account.collective) + res <- backend.tag.update(tag) + resp <- Ok(basicResult(res, "Tag successfully updated.")) } yield resp - case DELETE -> Root / Ident(id) => + case DELETE -> Root / Ident(id) => for { del <- backend.tag.delete(id, user.account.collective) resp <- Ok(basicResult(del, "Tag successfully deleted.")) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala index eb9bf6cc..c8972a3a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala @@ -26,9 +26,14 @@ object UploadRoutes { case req @ POST -> Root / "item" => for { multipart <- req.as[Multipart[F]] - updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes) - result <- backend.upload.submit(updata, user.account) - res <- Ok(basicResult(result)) + updata <- readMultipart( + multipart, + logger, + Priority.High, + cfg.backend.files.validMimeTypes + ) + result <- backend.upload.submit(updata, user.account) + res <- Ok(basicResult(result)) } yield res } @@ -39,12 +44,12 @@ object UploadRoutes { import dsl._ HttpRoutes.of { - case req @ POST -> Root / "item" / Ident(id)=> + case req @ POST -> Root / "item" / Ident(id) => for { multipart <- req.as[Multipart[F]] updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes) result <- backend.upload.submit(updata, id) - res <- Ok(basicResult(result)) + res <- Ok(basicResult(result)) } yield res } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala index bb6f154d..7a5d4c72 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala @@ -22,15 +22,19 @@ object UserRoutes { HttpRoutes.of { case req @ POST -> Root / "changePassword" => for { - data <- req.as[PasswordChange] - res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword) - resp <- Ok(basicResult(res)) + data <- req.as[PasswordChange] + res <- backend.collective.changePassword( + user.account, + data.currentPassword, + data.newPassword + ) + resp <- Ok(basicResult(res)) } yield resp case GET -> Root => for { - all <- backend.collective.listUser(user.account.collective) - res <- Ok(UserList(all.map(mkUser).toList)) + all <- backend.collective.listUser(user.account.collective) + res <- Ok(UserList(all.map(mkUser).toList)) } yield res case req @ POST -> Root => @@ -51,7 +55,7 @@ object UserRoutes { case DELETE -> Root / Ident(id) => for { - ar <- backend.collective.deleteUser(id, user.account.collective) + ar <- backend.collective.deleteUser(id, user.account.collective) resp <- Ok(basicResult(ar, "User deleted.")) } yield resp } diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala index f67cb6b3..8731c635 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala @@ -8,14 +8,21 @@ import docspell.backend.signup.{Config => SignupConfig} import yamusca.imports._ import yamusca.implicits._ -case class Flags( appName: String - , baseUrl: LenientUri - , signupMode: SignupConfig.Mode - , docspellAssetPath: String) +case class Flags( + appName: String, + baseUrl: LenientUri, + signupMode: SignupConfig.Mode, + docspellAssetPath: String +) object Flags { def apply(cfg: Config): Flags = - Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode, s"assets/docspell-webapp/${BuildInfo.version}") + Flags( + cfg.appName, + cfg.baseUrl, + cfg.backend.signup.mode, + s"assets/docspell-webapp/${BuildInfo.version}" + ) implicit val jsonEncoder: Encoder[Flags] = deriveEncoder[Flags] diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala index 5e734c8c..17d6779a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala @@ -21,90 +21,100 @@ object TemplateRoutes { val `text/html` = new MediaType("text", "html") - def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit C: ContextShift[F]): HttpRoutes[F] = { + def apply[F[_]: Effect](blocker: Blocker, cfg: Config)( + implicit C: ContextShift[F] + ): HttpRoutes[F] = { val indexTemplate = memo(loadResource("/index.html").flatMap(loadTemplate(_, blocker))) - val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker))) + val docTemplate = memo(loadResource("/doc.html").flatMap(loadTemplate(_, blocker))) - val dsl = new Http4sDsl[F]{} + val dsl = new Http4sDsl[F] {} import dsl._ HttpRoutes.of[F] { case GET -> Root / "index.html" => for { - templ <- indexTemplate - resp <- Ok(IndexData(cfg).render(templ), `Content-Type`(`text/html`)) + templ <- indexTemplate + resp <- Ok(IndexData(cfg).render(templ), `Content-Type`(`text/html`)) } yield resp case GET -> Root / "doc" => for { - templ <- docTemplate - resp <- Ok(DocData().render(templ), `Content-Type`(`text/html`)) + templ <- docTemplate + resp <- Ok(DocData().render(templ), `Content-Type`(`text/html`)) } yield resp } } - def loadResource[F[_]: Sync](name: String): F[URL] = { + def loadResource[F[_]: Sync](name: String): F[URL] = Option(getClass.getResource(name)) match { case None => - Sync[F].raiseError(new Exception("Unknown resource: "+ name)) + Sync[F].raiseError(new Exception("Unknown resource: " + name)) case Some(r) => r.pure[F] } - } def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[String] = - Stream.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())). - flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)). - through(text.utf8Decode). - compile.fold("")(_ + _) + Stream + .bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())) + .flatMap(in => io.readInputStream(in.pure[F], 64 * 1024, blocker, false)) + .through(text.utf8Decode) + .compile + .fold("")(_ + _) def parseTemplate[F[_]: Sync](str: String): F[Template] = Sync[F].delay { mustache.parse(str) match { - case Right(t) => t + case Right(t) => t case Left((_, err)) => sys.error(err) } } - def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit C: ContextShift[F]): F[Template] = { - loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)). - map(t => { - logger.info(s"Compiled template $url") - t - }) - } + def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)( + implicit C: ContextShift[F] + ): F[Template] = + loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t => + logger.info(s"Compiled template $url") + t + } case class DocData(swaggerRoot: String, openapiSpec: String) object DocData { def apply(): DocData = - DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml") + DocData( + "/app/assets" + Webjars.swaggerui, + s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml" + ) implicit def yamuscaValueConverter: ValueConverter[DocData] = ValueConverter.deriveConverter[DocData] } - case class IndexData(flags: Flags - , cssUrls: Seq[String] - , jsUrls: Seq[String] - , faviconBase: String - , appExtraJs: String - , flagsJson: String) + case class IndexData( + flags: Flags, + cssUrls: Seq[String], + jsUrls: Seq[String], + faviconBase: String, + appExtraJs: String, + flagsJson: String + ) object IndexData { def apply(cfg: Config): IndexData = - IndexData(Flags(cfg) - , Seq( + IndexData( + Flags(cfg), + Seq( "/app/assets" + Webjars.semanticui + "/semantic.min.css", s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.css" - ) - , Seq( + ), + Seq( "/app/assets" + Webjars.jquery + "/jquery.min.js", "/app/assets" + Webjars.semanticui + "/semantic.min.js", s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js" - ) - , s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon" - , s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js" - , Flags(cfg).asJson.spaces2 ) + ), + s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon", + s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js", + Flags(cfg).asJson.spaces2 + ) implicit def yamuscaValueConverter: ValueConverter[IndexData] = ValueConverter.deriveConverter[IndexData] @@ -116,10 +126,10 @@ object TemplateRoutes { Option(ref.get) match { case Some(a) => a.pure[F] case None => - fa.map(a => { + fa.map { a => ref.set(a) a - }) + } } } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala index 2becf5ea..6b370edd 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala @@ -9,7 +9,7 @@ import org.http4s.server.staticcontent.WebjarService.{WebjarAsset, Config => Web object WebjarRoutes { - def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = { + def appRoutes[F[_]: Effect](blocker: Blocker)(implicit C: ContextShift[F]): HttpRoutes[F] = webjarService( WebjarConfig( filter = assetFilter, @@ -17,10 +17,23 @@ object WebjarRoutes { cacheStrategy = NoopCacheStrategy[F] ) ) - } def assetFilter(asset: WebjarAsset): Boolean = - List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml"). - exists(e => asset.asset.endsWith(e)) + List( + ".js", + ".css", + ".html", + ".json", + ".jpg", + ".png", + ".eot", + ".woff", + ".woff2", + ".svg", + ".otf", + ".ttf", + ".yml", + ".xml" + ).exists(e => asset.asset.endsWith(e)) } diff --git a/modules/store/src/main/scala/docspell/store/AddResult.scala b/modules/store/src/main/scala/docspell/store/AddResult.scala index f1dc79de..79e313fd 100644 --- a/modules/store/src/main/scala/docspell/store/AddResult.scala +++ b/modules/store/src/main/scala/docspell/store/AddResult.scala @@ -18,14 +18,14 @@ object AddResult { e.fold(Failure, n => if (n > 0) Success else Failure(new Exception("No rows updated"))) case object Success extends AddResult { - def toEither = Right(()) + def toEither = Right(()) val isSuccess = true def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = fa(this) } case class EntityExists(msg: String) extends AddResult { - def toEither = Left(new Exception(msg)) + def toEither = Left(new Exception(msg)) val isSuccess = false def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = fb(this) @@ -35,7 +35,7 @@ object AddResult { } case class Failure(ex: Throwable) extends AddResult { - def toEither = Left(ex) + def toEither = Left(ex) val isSuccess = false def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = fc(this) diff --git a/modules/store/src/main/scala/docspell/store/JdbcConfig.scala b/modules/store/src/main/scala/docspell/store/JdbcConfig.scala index ff5960d5..13edbe21 100644 --- a/modules/store/src/main/scala/docspell/store/JdbcConfig.scala +++ b/modules/store/src/main/scala/docspell/store/JdbcConfig.scala @@ -2,10 +2,7 @@ package docspell.store import docspell.common.LenientUri -case class JdbcConfig(url: LenientUri - , user: String - , password: String -) { +case class JdbcConfig(url: LenientUri, user: String, password: String) { val dbmsName: Option[String] = JdbcConfig.extractDbmsName(url) diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index fbc541bd..bbf50a9e 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -22,21 +22,25 @@ trait Store[F[_]] { object Store { - def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig - , connectEC: ExecutionContext - , blocker: Blocker): Resource[F, Store[F]] = { + def create[F[_]: Effect: ContextShift]( + jdbc: JdbcConfig, + connectEC: ExecutionContext, + blocker: Blocker + ): Resource[F, Store[F]] = { - val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass - , jdbc.url.asString - , jdbc.user - , jdbc.password - , connectEC - , blocker) + val hxa = HikariTransactor.newHikariTransactor[F]( + jdbc.driverClass, + jdbc.url.asString, + jdbc.user, + jdbc.password, + connectEC, + blocker + ) for { - xa <- hxa - st = new StoreImpl[F](jdbc, xa) - _ <- Resource.liftF(st.migrate) + xa <- hxa + st = new StoreImpl[F](jdbc, xa) + _ <- Resource.liftF(st.migrate) } yield st } } diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala index 1fadcdbe..3d58d88a 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Column.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Column.scala @@ -24,7 +24,7 @@ case class Column(name: String, ns: String = "", alias: String = "") { def is[A: Put](ov: Option[A]): Fragment = ov match { case Some(v) => f ++ fr" = $v" - case None => fr"is null" + case None => fr"is null" } def is(c: Column): Fragment = @@ -42,7 +42,7 @@ case class Column(name: String, ns: String = "", alias: String = "") { def isOrDiscard[A: Put](value: Option[A]): Fragment = value match { case Some(v) => is(v) - case None => Fragment.empty + case None => Fragment.empty } def isOneOf[A: Put](values: Seq[A]): Fragment = { diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala index 96a83f3d..43372303 100644 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala @@ -21,7 +21,9 @@ trait DoobieMeta { }) def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = - Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces) + Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => + e.apply(a).noSpaces + ) implicit val metaCollectiveState: Meta[CollectiveState] = Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString) @@ -45,7 +47,9 @@ trait DoobieMeta { Meta[String].imap(JobState.unsafe)(_.name) implicit val metaDirection: Meta[Direction] = - Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d)) + Meta[Boolean].imap(flag => + if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction + )(d => Direction.isIncoming(d)) implicit val metaPriority: Meta[Priority] = Meta[Int].imap(Priority.fromInt)(Priority.toInt) diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala index 7ea864cc..47e61345 100644 --- a/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala @@ -19,7 +19,9 @@ trait DoobieSyntax { commas(fa :: fas.toList) def and(fs: Seq[Fragment]): Fragment = - Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") + Fragment.const(" (") ++ fs + .filter(f => !isEmpty(f)) + .reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") def and(f0: Fragment, fs: Fragment*): Fragment = and(f0 :: fs.toList) @@ -48,8 +50,9 @@ trait DoobieSyntax { def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment = Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ - commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")")) - + commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas( + vals.map(f => sql"(" ++ f ++ sql")") + ) def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = selectSimple(commas(cols.map(_.f)), table, where) @@ -62,7 +65,6 @@ trait DoobieSyntax { Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++ Fragment.const(") FROM ") ++ table ++ this.where(where) - // def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment = // selectSimple(cols.map(_.prefix("a")) // , table ++ fr"a," ++ RCollective.table ++ fr"b" @@ -70,11 +72,12 @@ trait DoobieSyntax { // else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b"))) def selectCount(col: Column, table: Fragment, where: Fragment): Fragment = - Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where) + Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where( + where + ) - def deleteFrom(table: Fragment, where: Fragment): Fragment = { + def deleteFrom(table: Fragment, where: Fragment): Fragment = fr"DELETE FROM" ++ table ++ this.where(where) - } def withCTE(ps: (String, Fragment)*): Fragment = { val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")") diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala index ac357814..9e18b3e1 100644 --- a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala +++ b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala @@ -1,5 +1,3 @@ package docspell.store.impl - -object Implicits extends DoobieMeta - with DoobieSyntax +object Implicits extends DoobieMeta with DoobieSyntax diff --git a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala index 8889abc4..3f991f56 100644 --- a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala +++ b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala @@ -10,7 +10,8 @@ import doobie._ import doobie.implicits._ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] { - val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id)) + val bitpeaceCfg = + BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id)) def migrate: F[Int] = FlywayMigrate.run[F](jdbc) @@ -24,14 +25,14 @@ final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends def bitpeace: Bitpeace[F] = Bitpeace(bitpeaceCfg, xa) - def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = { + def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = for { save <- transact(insert).attempt exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b))) } yield exist.swap match { case Right(_) => AddResult.Success - case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.") + case Left((_, true)) => + AddResult.EntityExists("Adding failed, because the entity already exists.") case Left((ex, _)) => AddResult.Failure(ex) } - } } diff --git a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala index 5c327f3b..5fbcac31 100644 --- a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala +++ b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala @@ -20,11 +20,12 @@ object FlywayMigrate { } logger.info(s"Using migration locations: $locations") - val fw = Flyway.configure(). - cleanDisabled(true). - dataSource(jdbc.url.asString, jdbc.user, jdbc.password). - locations(locations: _*). - load() + val fw = Flyway + .configure() + .cleanDisabled(true) + .dataSource(jdbc.url.asString, jdbc.user, jdbc.password) + .locations(locations: _*) + .load() fw.repair() fw.migrate() diff --git a/modules/store/src/main/scala/docspell/store/ops/ONode.scala b/modules/store/src/main/scala/docspell/store/ops/ONode.scala index 2d18a6e5..42f8c91e 100644 --- a/modules/store/src/main/scala/docspell/store/ops/ONode.scala +++ b/modules/store/src/main/scala/docspell/store/ops/ONode.scala @@ -18,7 +18,7 @@ trait ONode[F[_]] { object ONode { private[this] val logger = getLogger - def apply[F[_] : Effect](store: Store[F]): Resource[F, ONode[F]] = + def apply[F[_]: Effect](store: Store[F]): Resource[F, ONode[F]] = Resource.pure(new ONode[F] { def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala index 9b9f7731..9a652df0 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -12,58 +12,64 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} object QAttachment { - def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = { + def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = for { raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)) n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId))) - f <- Stream.emit(raOpt). - unNoneTerminate. - map(_.fileId.id). - flatMap(store.bitpeace.delete). - compile.last + f <- Stream + .emit(raOpt) + .unNoneTerminate + .map(_.fileId.id) + .flatMap(store.bitpeace.delete) + .compile + .last } yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0) - } - def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = { + def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = for { - n <- store.transact(RAttachment.delete(ra.id)) - f <- Stream.emit(ra.fileId.id). - flatMap(store.bitpeace.delete). - compile.last + n <- store.transact(RAttachment.delete(ra.id)) + f <- Stream.emit(ra.fileId.id).flatMap(store.bitpeace.delete).compile.last } yield n + f.map(_ => 1).getOrElse(0) - } - def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = { + def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = for { - ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll)) - ns <- ras.traverse(deleteAttachment[F](store)) - } yield ns.sum - } + ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll)) + ns <- ras.traverse(deleteAttachment[F](store)) + } yield ns.sum def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = { val AC = RAttachment.Columns val MC = RAttachmentMeta.Columns val IC = RItem.Columns - val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ + val q = fr"SELECT" ++ MC.proposals + .prefix("m") + .f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll)) for { - ml <- q.query[MetaProposalList].to[Vector] + ml <- q.query[MetaProposalList].to[Vector] } yield MetaProposalList.flatten(ml) } - def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = { + def getAttachmentMeta( + attachId: Ident, + collective: Ident + ): ConnectionIO[Option[RAttachmentMeta]] = { val AC = RAttachment.Columns val MC = RAttachmentMeta.Columns val IC = RItem.Columns val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ - fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ - fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ - fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective) + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id + .prefix("i") + .is(AC.itemId.prefix("a")) ++ + fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id + .prefix("a") + .is(MC.id.prefix("m")) ++ + fr"WHERE" ++ and(AC.id.prefix("a").is(attachId), IC.cid.prefix("i").is(collective)) q.query[RAttachmentMeta].option } diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala index 4e3661a2..1de40050 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -8,27 +8,35 @@ import docspell.store.records.{RAttachment, RItem, RTag, RTagItem} object QCollective { - case class InsightData( incoming: Int - , outgoing: Int - , bytes: Long - , tags: Map[String, Int]) + case class InsightData(incoming: Int, outgoing: Int, bytes: Long, tags: Map[String, Int]) def getInsights(coll: Ident): ConnectionIO[InsightData] = { val IC = RItem.Columns val AC = RAttachment.Columns val TC = RTag.Columns val RC = RTagItem.Columns - val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)). - query[Int].unique - val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)). - query[Int].unique + val q0 = selectCount( + IC.id, + RItem.table, + and(IC.cid.is(coll), IC.incoming.is(Direction.incoming)) + ).query[Int].unique + val q1 = selectCount( + IC.id, + RItem.table, + and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing)) + ).query[Int].unique val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++ - fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId + .prefix("a") + .is(IC.id.prefix("i")) ++ fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++ fr"WHERE" ++ IC.cid.is(coll) - val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++ + val q3 = fr"SELECT" ++ commas( + TC.name.prefix("t").f, + fr"count(" ++ RC.itemId.prefix("r").f ++ fr")" + ) ++ fr"FROM" ++ RTagItem.table ++ fr"r" ++ fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++ fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index f314a7fc..35d4dc5e 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -13,177 +13,227 @@ import docspell.store.impl.Implicits._ import org.log4s._ object QItem { - private [this] val logger = getLogger + private[this] val logger = getLogger - case class ItemData( item: RItem - , corrOrg: Option[ROrganization] - , corrPerson: Option[RPerson] - , concPerson: Option[RPerson] - , concEquip: Option[REquipment] - , inReplyTo: Option[IdRef] - , tags: Vector[RTag] - , attachments: Vector[(RAttachment, FileMeta)]) { + case class ItemData( + item: RItem, + corrOrg: Option[ROrganization], + corrPerson: Option[RPerson], + concPerson: Option[RPerson], + concEquip: Option[REquipment], + inReplyTo: Option[IdRef], + tags: Vector[RTag], + attachments: Vector[(RAttachment, FileMeta)] + ) { def filterCollective(coll: Ident): Option[ItemData] = if (item.cid == coll) Some(this) else None } def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { - val IC = RItem.Columns.all.map(_.prefix("i")) - val OC = ROrganization.Columns.all.map(_.prefix("o")) + val IC = RItem.Columns.all.map(_.prefix("i")) + val OC = ROrganization.Columns.all.map(_.prefix("o")) val P0C = RPerson.Columns.all.map(_.prefix("p0")) val P1C = RPerson.Columns.all.map(_.prefix("p1")) - val EC = REquipment.Columns.all.map(_.prefix("e")) + val EC = REquipment.Columns.all.map(_.prefix("e")) val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++ - fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg.prefix("i").is(ROrganization.Columns.oid.prefix("o")) ++ - fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson.prefix("i").is(RPerson.Columns.pid.prefix("p0")) ++ - fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson.prefix("i").is(RPerson.Columns.pid.prefix("p1")) ++ - fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment.prefix("i").is(REquipment.Columns.eid.prefix("e")) ++ - fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo.prefix("i").is(RItem.Columns.id.prefix("ref")) ++ + fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg + .prefix("i") + .is(ROrganization.Columns.oid.prefix("o")) ++ + fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson + .prefix("i") + .is(RPerson.Columns.pid.prefix("p0")) ++ + fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson + .prefix("i") + .is(RPerson.Columns.pid.prefix("p1")) ++ + fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment + .prefix("i") + .is(REquipment.Columns.eid.prefix("e")) ++ + fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo + .prefix("i") + .is(RItem.Columns.id.prefix("ref")) ++ fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) - val q = cq.query[(RItem, Option[ROrganization], Option[RPerson], Option[RPerson], Option[REquipment], Option[IdRef])].option + val q = cq + .query[ + ( + RItem, + Option[ROrganization], + Option[RPerson], + Option[RPerson], + Option[REquipment], + Option[IdRef] + ) + ] + .option val attachs = RAttachment.findByItemWithMeta(id) val tags = RTag.findByItem(id) for { - data <- q - att <- attachs - ts <- tags + data <- q + att <- attachs + ts <- tags } yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att)) } + case class ListItem( + id: Ident, + name: String, + state: ItemState, + date: Timestamp, + dueDate: Option[Timestamp], + source: String, + direction: Direction, + created: Timestamp, + fileCount: Int, + corrOrg: Option[IdRef], + corrPerson: Option[IdRef], + concPerson: Option[IdRef], + concEquip: Option[IdRef] + ) - case class ListItem( id: Ident - , name: String - , state: ItemState - , date: Timestamp - , dueDate: Option[Timestamp] - , source: String - , direction: Direction - , created: Timestamp - , fileCount: Int - , corrOrg: Option[IdRef] - , corrPerson: Option[IdRef] - , concPerson: Option[IdRef] - , concEquip: Option[IdRef]) - - case class Query( collective: Ident - , name: Option[String] - , states: Seq[ItemState] - , direction: Option[Direction] - , corrPerson: Option[Ident] - , corrOrg: Option[Ident] - , concPerson: Option[Ident] - , concEquip: Option[Ident] - , tagsInclude: List[Ident] - , tagsExclude: List[Ident] - , dateFrom: Option[Timestamp] - , dateTo: Option[Timestamp] - , dueDateFrom: Option[Timestamp] - , dueDateTo: Option[Timestamp]) + case class Query( + collective: Ident, + name: Option[String], + states: Seq[ItemState], + direction: Option[Direction], + corrPerson: Option[Ident], + corrOrg: Option[Ident], + concPerson: Option[Ident], + concEquip: Option[Ident], + tagsInclude: List[Ident], + tagsExclude: List[Ident], + dateFrom: Option[Timestamp], + dateTo: Option[Timestamp], + dueDateFrom: Option[Timestamp], + dueDateTo: Option[Timestamp] + ) def findItems(q: Query): Stream[ConnectionIO, ListItem] = { - val IC = RItem.Columns - val AC = RAttachment.Columns - val PC = RPerson.Columns - val OC = ROrganization.Columns - val EC = REquipment.Columns - val itemCols = IC.all + val IC = RItem.Columns + val AC = RAttachment.Columns + val PC = RPerson.Columns + val OC = ROrganization.Columns + val EC = REquipment.Columns + val itemCols = IC.all val personCols = List(RPerson.Columns.pid, RPerson.Columns.name) - val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name) - val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name) + val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name) + val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name) - val finalCols = commas(IC.id.prefix("i").f - , IC.name.prefix("i").f - , IC.state.prefix("i").f - , coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) - , IC.dueDate.prefix("i").f - , IC.source.prefix("i").f - , IC.incoming.prefix("i").f - , IC.created.prefix("i").f - , fr"COALESCE(a.num, 0)" - , OC.oid.prefix("o0").f - , OC.name.prefix("o0").f - , PC.pid.prefix("p0").f - , PC.name.prefix("p0").f - , PC.pid.prefix("p1").f - , PC.name.prefix("p1").f - , EC.eid.prefix("e1").f - , EC.name.prefix("e1").f + val finalCols = commas( + IC.id.prefix("i").f, + IC.name.prefix("i").f, + IC.state.prefix("i").f, + coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f), + IC.dueDate.prefix("i").f, + IC.source.prefix("i").f, + IC.incoming.prefix("i").f, + IC.created.prefix("i").f, + fr"COALESCE(a.num, 0)", + OC.oid.prefix("o0").f, + OC.name.prefix("o0").f, + PC.pid.prefix("p0").f, + PC.name.prefix("p0").f, + PC.pid.prefix("p1").f, + PC.name.prefix("p1").f, + EC.eid.prefix("e1").f, + EC.name.prefix("e1").f ) - val withItem = selectSimple(itemCols, RItem.table, IC.cid is q.collective) - val withPerson = selectSimple(personCols, RPerson.table, PC.cid is q.collective) - val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid is q.collective) - val withEquips = selectSimple(equipCols, REquipment.table, EC.cid is q.collective) + val withItem = selectSimple(itemCols, RItem.table, IC.cid.is(q.collective)) + val withPerson = selectSimple(personCols, RPerson.table, PC.cid.is(q.collective)) + val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid.is(q.collective)) + val withEquips = selectSimple(equipCols, REquipment.table, EC.cid.is(q.collective)) val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ - fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" + fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" - val query = withCTE("items" -> withItem - , "persons" -> withPerson - , "orgs" -> withOrgs - , "equips" -> withEquips - , "attachs" -> withAttach) ++ - fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++ - fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ - fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++ - fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++ - fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++ - fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1")) // i.concequipment = e1.eid" + val query = withCTE( + "items" -> withItem, + "persons" -> withPerson, + "orgs" -> withOrgs, + "equips" -> withEquips, + "attachs" -> withAttach + ) ++ + fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++ + fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ + fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson + .prefix("i") + .is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++ + fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg + .prefix("i") + .is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++ + fr"LEFT JOIN persons p1 ON" ++ IC.concPerson + .prefix("i") + .is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++ + fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment + .prefix("i") + .is(EC.eid.prefix("e1")) // i.concequipment = e1.eid" // inclusive tags are AND-ed - val tagSelectsIncl = q.tagsInclude.map(tid => - selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId is tid)). - map(f => sql"(" ++ f ++ sql") ") + val tagSelectsIncl = q.tagsInclude + .map(tid => + selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId.is(tid)) + ) + .map(f => sql"(" ++ f ++ sql") ") // exclusive tags are OR-ed val tagSelectsExcl = if (q.tagsExclude.isEmpty) Fragment.empty - else selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId isOneOf q.tagsExclude) + else + selectSimple( + List(RTagItem.Columns.itemId), + RTagItem.table, + RTagItem.Columns.tagId.isOneOf(q.tagsExclude) + ) val name = q.name.map(queryWildcard) val cond = and( - IC.cid.prefix("i") is q.collective, - IC.state.prefix("i") isOneOf q.states, - IC.incoming.prefix("i") isOrDiscard q.direction, + IC.cid.prefix("i").is(q.collective), + IC.state.prefix("i").isOneOf(q.states), + IC.incoming.prefix("i").isOrDiscard(q.direction), name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty), - RPerson.Columns.pid.prefix("p0") isOrDiscard q.corrPerson, - ROrganization.Columns.oid.prefix("o0") isOrDiscard q.corrOrg, - RPerson.Columns.pid.prefix("p1") isOrDiscard q.concPerson, - REquipment.Columns.eid.prefix("e1") isOrDiscard q.concEquip, + RPerson.Columns.pid.prefix("p0").isOrDiscard(q.corrPerson), + ROrganization.Columns.oid.prefix("o0").isOrDiscard(q.corrOrg), + RPerson.Columns.pid.prefix("p1").isOrDiscard(q.concPerson), + REquipment.Columns.eid.prefix("e1").isOrDiscard(q.concEquip), if (q.tagsInclude.isEmpty) Fragment.empty - else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", + else + IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", if (q.tagsExclude.isEmpty) Fragment.empty else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")", - q.dateFrom.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d").getOrElse(Fragment.empty), - q.dateTo.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d").getOrElse(Fragment.empty), + q.dateFrom + .map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d") + .getOrElse(Fragment.empty), + q.dateTo + .map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d") + .getOrElse(Fragment.empty), q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty), q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty) ) val order = orderBy(coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"DESC") - val frag = query ++ fr"WHERE" ++ cond ++ order + val frag = query ++ fr"WHERE" ++ cond ++ order logger.trace(s"List items: $frag") frag.query[ListItem].stream } - def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = for { - tn <- store.transact(RTagItem.deleteItemTags(itemId)) - rn <- QAttachment.deleteItemAttachments(store)(itemId, collective) - n <- store.transact(RItem.deleteByIdAndCollective(itemId, collective)) + tn <- store.transact(RTagItem.deleteItemTags(itemId)) + rn <- QAttachment.deleteItemAttachments(store)(itemId, collective) + n <- store.transact(RItem.deleteByIdAndCollective(itemId, collective)) } yield tn + rn + n def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = { val IC = RItem.Columns val AC = RAttachment.Columns val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ - fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId + .prefix("a") + .is(IC.id.prefix("i")) ++ fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc) q.query[RItem].to[Vector] diff --git a/modules/store/src/main/scala/docspell/store/queries/QJob.scala b/modules/store/src/main/scala/docspell/store/queries/QJob.scala index c5be9dda..646022d8 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QJob.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QJob.scala @@ -13,90 +13,122 @@ import fs2.Stream import org.log4s._ object QJob { - private [this] val logger = getLogger + private[this] val logger = getLogger - def takeNextJob[F[_]: Effect](store: Store[F])(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = { - Stream.range(0, 10). - evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)). - evalTap({ x => - if (x.isLeft) logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.") + def takeNextJob[F[_]: Effect]( + store: Store[F] + )(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = + Stream + .range(0, 10) + .evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)) + .evalTap({ x => + if (x.isLeft) + logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.") else ().pure[F] - }). - find(_.isRight). - flatMap({ + }) + .find(_.isRight) + .flatMap({ case Right(job) => Stream.emit(job) case Left(_) => - Stream.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")).map(_ => None) - }). - compile.last.map(_.flatten) - } + Stream + .eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")) + .map(_ => None) + }) + .compile + .last + .map(_.flatten) - private def takeNextJob1[F[_]: Effect](store: Store[F])( priority: Ident => F[Priority] - , worker: Ident - , retryPause: Duration - , currentTry: Int): F[Either[Unit, Option[RJob]]] = { + private def takeNextJob1[F[_]: Effect](store: Store[F])( + priority: Ident => F[Priority], + worker: Ident, + retryPause: Duration, + currentTry: Int + ): F[Either[Unit, Option[RJob]]] = { //if this fails, we have to restart takeNextJob def markJob(job: RJob): F[Either[Unit, RJob]] = store.transact(for { - n <- RJob.setScheduled(job.id, worker) - _ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group)) - else 0.pure[ConnectionIO] + n <- RJob.setScheduled(job.id, worker) + _ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group)) + else 0.pure[ConnectionIO] } yield if (n == 1) Right(job) else Left(())) for { - _ <- logger.ftrace[F](s"About to take next job (worker ${worker.id}), try $currentTry") - now <- Timestamp.current[F] - group <- store.transact(selectNextGroup(worker, now, retryPause)) - _ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}") - prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F]) - _ <- logger.ftrace[F](s"Looking for job of prio $prio") - job <- group.map(g => store.transact(selectNextJob(g, prio, retryPause, now))).getOrElse((None: Option[RJob]).pure[F]) - _ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}") - res <- job.traverse(j => markJob(j)) + _ <- logger.ftrace[F](s"About to take next job (worker ${worker.id}), try $currentTry") + now <- Timestamp.current[F] + group <- store.transact(selectNextGroup(worker, now, retryPause)) + _ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}") + prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F]) + _ <- logger.ftrace[F](s"Looking for job of prio $prio") + job <- group + .map(g => store.transact(selectNextJob(g, prio, retryPause, now))) + .getOrElse((None: Option[RJob]).pure[F]) + _ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}") + res <- job.traverse(j => markJob(j)) } yield res.map(_.map(_.some)).getOrElse { if (group.isDefined) Left(()) // if a group was found, but no job someone else was faster else Right(None) } } - def selectNextGroup(worker: Ident, now: Timestamp, initialPause: Duration): ConnectionIO[Option[Ident]] = { - val JC = RJob.Columns + def selectNextGroup( + worker: Ident, + now: Timestamp, + initialPause: Duration + ): ConnectionIO[Option[Ident]] = { + val JC = RJob.Columns val waiting: JobState = JobState.Waiting - val stuck: JobState = JobState.Stuck - val jgroup = JC.group.prefix("a") - val jstate = JC.state.prefix("a") - val ugroup = RJobGroupUse.Columns.group.prefix("b") - val uworker = RJobGroupUse.Columns.worker.prefix("b") + val stuck: JobState = JobState.Stuck + val jgroup = JC.group.prefix("a") + val jstate = JC.state.prefix("a") + val ugroup = RJobGroupUse.Columns.group.prefix("b") + val uworker = RJobGroupUse.Columns.worker.prefix("b") val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}" - val stateCond = or(jstate is waiting, and(jstate is stuck, stuckTrigger ++ fr"< ${now.toMillis}")) + val stateCond = + or(jstate.is(waiting), and(jstate.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}")) val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++ fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++ - fr"WHERE" ++ and(uworker is worker, stateCond) ++ + fr"WHERE" ++ and(uworker.is(worker), stateCond) ++ fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++ fr"WHERE" ++ stateCond val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null" - union.query[Ident].to[List].map(_.headOption) // either one or two results, but may be empty if RJob table is empty + union + .query[Ident] + .to[List] + .map(_.headOption) // either one or two results, but may be empty if RJob table is empty } - def selectNextJob(group: Ident, prio: Priority, initialPause: Duration, now: Timestamp): ConnectionIO[Option[RJob]] = { + def selectNextJob( + group: Ident, + prio: Priority, + initialPause: Duration, + now: Timestamp + ): ConnectionIO[Option[RJob]] = { val JC = RJob.Columns val psort = if (prio == Priority.High) JC.priority.desc else JC.priority.asc val waiting: JobState = JobState.Waiting - val stuck: JobState = JobState.Stuck + val stuck: JobState = JobState.Stuck - val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries) ++ fr"* ${initialPause.millis}" - val sql = selectSimple(JC.all, RJob.table, - and(JC.group is group, or(JC.state is waiting, and(JC.state is stuck, stuckTrigger ++ fr"< ${now.toMillis}")))) ++ + val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2( + JC.retries + ) ++ fr"* ${initialPause.millis}" + val sql = selectSimple( + JC.all, + RJob.table, + and( + JC.group.is(group), + or(JC.state.is(waiting), and(JC.state.is(stuck), stuckTrigger ++ fr"< ${now.toMillis}")) + ) + ) ++ orderBy(JC.state.asc, psort, JC.submitted.asc) ++ fr"LIMIT 1" @@ -150,25 +182,31 @@ object QJob { def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] = store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max)) - def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = { + def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ()) - } def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] = store.transact(RJob.findFromIds(ids)) def queueStateSnapshot(collective: Ident): Stream[ConnectionIO, (RJob, Vector[RJobLog])] = { - val JC = RJob.Columns + val JC = RJob.Columns val waiting: Set[JobState] = Set(JobState.Waiting, JobState.Stuck, JobState.Scheduled) val running: Set[JobState] = Set(JobState.Running) - val done = JobState.all.diff(waiting).diff(running) + val done = JobState.all.diff(waiting).diff(running) def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = { val refDate = now.minusHours(24) - val sql = selectSimple(JC.all, RJob.table, - and(JC.group is collective, - or(and(JC.state.isOneOf(done.toSeq), JC.submitted isGt refDate) - , JC.state.isOneOf((running ++ waiting).toSeq)))) + val sql = selectSimple( + JC.all, + RJob.table, + and( + JC.group.is(collective), + or( + and(JC.state.isOneOf(done.toSeq), JC.submitted.isGt(refDate)), + JC.state.isOneOf((running ++ waiting).toSeq) + ) + ) + ) (sql ++ orderBy(JC.submitted.desc)).query[RJob].stream } @@ -176,9 +214,9 @@ object QJob { RJobLog.findLogs(job.id) for { - now <- Stream.eval(Timestamp.current[ConnectionIO]) - job <- selectJobs(now) - res <- Stream.eval(selectLogs(job)) + now <- Stream.eval(Timestamp.current[ConnectionIO]) + job <- selectJobs(now) + res <- Stream.eval(selectLogs(job)) } yield (job, res) } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala index 4525c6d9..7c114a76 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala @@ -12,23 +12,26 @@ import org.log4s._ object QLogin { private[this] val logger = getLogger - case class Data( account: AccountId - , password: Password - , collectiveState: CollectiveState - , userState: UserState) + case class Data( + account: AccountId, + password: Password, + collectiveState: CollectiveState, + userState: UserState + ) def findUser(acc: AccountId): ConnectionIO[Option[Data]] = { - val ucid = UC.cid.prefix("u") - val login = UC.login.prefix("u") - val pass = UC.password.prefix("u") + val ucid = UC.cid.prefix("u") + val login = UC.login.prefix("u") + val pass = UC.password.prefix("u") val ustate = UC.state.prefix("u") val cstate = CC.state.prefix("c") - val ccid = CC.id.prefix("c") + val ccid = CC.id.prefix("c") val sql = selectSimple( - List(ucid,login,pass,cstate,ustate), + List(ucid, login, pass, cstate, ustate), RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c", - and(ucid is ccid, login is acc.user, ucid is acc.collective)) + and(ucid.is(ccid), login.is(acc.user), ucid.is(acc.collective)) + ) logger.trace(s"SQL : $sql") sql.query[Data].option diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala index 361c5338..bb89bc30 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala @@ -12,19 +12,27 @@ import docspell.store.records._ object QOrganization { - def findOrgAndContact(coll: Ident, order: OC.type => Column): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = { - ROrganization.findAll(coll, order). - evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs))) - } - def findPersonAndContact(coll: Ident, order: PC.type => Column): Stream[ConnectionIO, (RPerson, Vector[RContact])] = { - RPerson.findAll(coll, order). - evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs))) - } + def findOrgAndContact( + coll: Ident, + order: OC.type => Column + ): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = + ROrganization + .findAll(coll, order) + .evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs))) + def findPersonAndContact( + coll: Ident, + order: PC.type => Column + ): Stream[ConnectionIO, (RPerson, Vector[RContact])] = + RPerson.findAll(coll, order).evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs))) - def addOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + def addOrg[F[_]]( + org: ROrganization, + contacts: Seq[RContact], + cid: Ident + ): Store[F] => F[AddResult] = { val insert = for { - n <- ROrganization.insert(org) - cs <- contacts.toList.traverse(RContact.insert) + n <- ROrganization.insert(org) + cs <- contacts.toList.traverse(RContact.insert) } yield n + cs.sum val exists = ROrganization.existsByName(cid, org.name) @@ -32,10 +40,14 @@ object QOrganization { store => store.add(insert, exists) } - def addPerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + def addPerson[F[_]]( + person: RPerson, + contacts: Seq[RContact], + cid: Ident + ): Store[F] => F[AddResult] = { val insert = for { - n <- RPerson.insert(person) - cs <- contacts.toList.traverse(RContact.insert) + n <- RPerson.insert(person) + cs <- contacts.toList.traverse(RContact.insert) } yield n + cs.sum val exists = RPerson.existsByName(cid, person.name) @@ -43,11 +55,15 @@ object QOrganization { store => store.add(insert, exists) } - def updateOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + def updateOrg[F[_]]( + org: ROrganization, + contacts: Seq[RContact], + cid: Ident + ): Store[F] => F[AddResult] = { val insert = for { - n <- ROrganization.update(org) - d <- RContact.deleteOrg(org.oid) - cs <- contacts.toList.traverse(RContact.insert) + n <- ROrganization.update(org) + d <- RContact.deleteOrg(org.oid) + cs <- contacts.toList.traverse(RContact.insert) } yield n + cs.sum + d val exists = ROrganization.existsByName(cid, org.name) @@ -55,11 +71,15 @@ object QOrganization { store => store.add(insert, exists) } - def updatePerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + def updatePerson[F[_]]( + person: RPerson, + contacts: Seq[RContact], + cid: Ident + ): Store[F] => F[AddResult] = { val insert = for { - n <- RPerson.update(person) - d <- RContact.deletePerson(person.pid) - cs <- contacts.toList.traverse(RContact.insert) + n <- RPerson.update(person) + d <- RContact.deletePerson(person.pid) + cs <- contacts.toList.traverse(RContact.insert) } yield n + cs.sum + d val exists = RPerson.existsByName(cid, person.name) @@ -67,20 +87,18 @@ object QOrganization { store => store.add(insert, exists) } - def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = { + def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = for { - n0 <- RItem.removeCorrOrg(collective, orgId) - n1 <- RContact.deleteOrg(orgId) - n2 <- ROrganization.delete(orgId, collective) + n0 <- RItem.removeCorrOrg(collective, orgId) + n1 <- RContact.deleteOrg(orgId) + n2 <- ROrganization.delete(orgId, collective) } yield n0 + n1 + n2 - } - def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = { + def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = for { - n0 <- RItem.removeCorrPerson(collective, personId) - n1 <- RItem.removeConcPerson(collective, personId) - n2 <- RContact.deletePerson(personId) - n3 <- RPerson.delete(personId, collective) + n0 <- RItem.removeCorrPerson(collective, personId) + n1 <- RItem.removeConcPerson(collective, personId) + n2 <- RContact.deletePerson(personId) + n3 <- RPerson.delete(personId, collective) } yield n0 + n1 + n2 + n3 - } } diff --git a/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala b/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala index 8134e5a1..25607126 100644 --- a/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala +++ b/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala @@ -21,22 +21,29 @@ trait JobQueue[F[_]] { object JobQueue { private[this] val logger = getLogger - def apply[F[_] : Effect](store: Store[F]): Resource[F, JobQueue[F]] = + def apply[F[_]: Effect](store: Store[F]): Resource[F, JobQueue[F]] = Resource.pure(new JobQueue[F] { - def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = + def nextJob( + prio: Ident => F[Priority], + worker: Ident, + retryPause: Duration + ): F[Option[RJob]] = logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause) def insert(job: RJob): F[Unit] = - store.transact(RJob.insert(job)). - flatMap({ n => - if (n != 1) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n")) + store + .transact(RJob.insert(job)) + .flatMap({ n => + if (n != 1) + Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n")) else ().pure[F] }) def insertAll(jobs: Seq[RJob]): F[Unit] = - jobs.toList.traverse(j => insert(j).attempt). - map(_.foreach { + jobs.toList + .traverse(j => insert(j).attempt) + .map(_.foreach { case Right(()) => case Left(ex) => logger.error(ex)("Could not insert job. Skipping it.") diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index 5399d929..52f71d30 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -7,50 +7,59 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class RAttachment( id: Ident - , itemId: Ident - , fileId: Ident - , position: Int - , created: Timestamp - , name: Option[String]) { - -} +case class RAttachment( + id: Ident, + itemId: Ident, + fileId: Ident, + position: Int, + created: Timestamp, + name: Option[String] +) {} object RAttachment { val table = fr"attachment" object Columns { - val id = Column("attachid") - val itemId = Column("itemid") - val fileId = Column("filemetaid") + val id = Column("attachid") + val itemId = Column("itemid") + val fileId = Column("filemetaid") val position = Column("position") - val created = Column("created") - val name = Column("name") - val all = List(id, itemId, fileId, position, created, name) + val created = Column("created") + val name = Column("name") + val all = List(id, itemId, fileId, position, created, name) } import Columns._ def insert(v: RAttachment): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}").update.run + insertRow( + table, + all, + fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}" + ).update.run def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] = - selectSimple(all, table, id is attachId).query[RAttachment].option + selectSimple(all, table, id.is(attachId)).query[RAttachment].option - def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = { - selectSimple(all.map(_.prefix("a")), table ++ fr"a," ++ RItem.table ++ fr"i", and( - fr"a.itemid = i.itemid", - id.prefix("a") is attachId, - RItem.Columns.cid.prefix("i") is collective - )).query[RAttachment].option - } + def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = + selectSimple( + all.map(_.prefix("a")), + table ++ fr"a," ++ RItem.table ++ fr"i", + and( + fr"a.itemid = i.itemid", + id.prefix("a").is(attachId), + RItem.Columns.cid.prefix("i").is(collective) + ) + ).query[RAttachment].option def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] = - selectSimple(all, table, itemId is id).query[RAttachment].to[Vector] + selectSimple(all, table, itemId.is(id)).query[RAttachment].to[Vector] def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = { val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ - fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id.prefix("i").is(itemId.prefix("a")) ++ + fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id + .prefix("i") + .is(itemId.prefix("a")) ++ fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) q.query[RAttachment].to[Vector] } @@ -65,7 +74,7 @@ object RAttachment { def delete(attachId: Ident): ConnectionIO[Int] = for { n0 <- RAttachmentMeta.delete(attachId) - n1 <- deleteFrom(table, id is attachId).update.run + n1 <- deleteFrom(table, id.is(attachId)).update.run } yield n0 + n1 } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala index 27f4fc55..f1887399 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -7,12 +7,12 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class RAttachmentMeta(id: Ident - , content: Option[String] - , nerlabels: List[NerLabel] - , proposals: MetaProposalList) { - -} +case class RAttachmentMeta( + id: Ident, + content: Option[String], + nerlabels: List[NerLabel], + proposals: MetaProposalList +) {} object RAttachmentMeta { def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty) @@ -20,11 +20,11 @@ object RAttachmentMeta { val table = fr"attachmentmeta" object Columns { - val id = Column("attachid") - val content = Column("content") + val id = Column("attachid") + val content = Column("content") val nerlabels = Column("nerlabels") val proposals = Column("itemproposals") - val all = List(id, content, nerlabels, proposals) + val all = List(id, content, nerlabels, proposals) } import Columns._ @@ -32,31 +32,43 @@ object RAttachmentMeta { insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run def exists(attachId: Ident): ConnectionIO[Boolean] = - selectCount(id, table, id is attachId).query[Int].unique.map(_ > 0) + selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0) def upsert(v: RAttachmentMeta): ConnectionIO[Int] = for { - n0 <- update(v) - n1 <- if (n0 == 0) insert(v) else n0.pure[ConnectionIO] + n0 <- update(v) + n1 <- if (n0 == 0) insert(v) else n0.pure[ConnectionIO] } yield n1 def update(v: RAttachmentMeta): ConnectionIO[Int] = - updateRow(table, id is v.id, commas( - content setTo v.content, - nerlabels setTo v.nerlabels, - proposals setTo v.proposals - )).update.run + updateRow( + table, + id.is(v.id), + commas( + content.setTo(v.content), + nerlabels.setTo(v.nerlabels), + proposals.setTo(v.proposals) + ) + ).update.run def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] = - updateRow(table, id is mid, commas( - nerlabels setTo labels - )).update.run + updateRow( + table, + id.is(mid), + commas( + nerlabels.setTo(labels) + ) + ).update.run def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] = - updateRow(table, id is mid, commas( - proposals setTo plist - )).update.run + updateRow( + table, + id.is(mid), + commas( + proposals.setTo(plist) + ) + ).update.run def delete(attachId: Ident): ConnectionIO[Int] = - deleteFrom(table, id is attachId).update.run + deleteFrom(table, id.is(attachId)).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RCollective.scala b/modules/store/src/main/scala/docspell/store/records/RCollective.scala index 1d32b647..7aaf526a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RCollective.scala +++ b/modules/store/src/main/scala/docspell/store/records/RCollective.scala @@ -7,10 +7,7 @@ import doobie._ import doobie.implicits._ import fs2.Stream -case class RCollective( id: Ident - , state: CollectiveState - , language: Language - , created: Timestamp) +case class RCollective(id: Ident, state: CollectiveState, language: Language, created: Timestamp) object RCollective { @@ -18,10 +15,10 @@ object RCollective { object Columns { - val id = Column("cid") - val state = Column("state") + val id = Column("cid") + val state = Column("state") val language = Column("doclang") - val created = Column("created") + val created = Column("created") val all = List(id, state, language, created) } @@ -29,30 +26,38 @@ object RCollective { import Columns._ def insert(value: RCollective): ConnectionIO[Int] = { - val sql = insertRow(table, Columns.all, fr"${value.id},${value.state},${value.language},${value.created}") + val sql = insertRow( + table, + Columns.all, + fr"${value.id},${value.state},${value.language},${value.created}" + ) sql.update.run } def update(value: RCollective): ConnectionIO[Int] = { - val sql = updateRow(table, id is value.id, commas( - state setTo value.state - )) + val sql = updateRow( + table, + id.is(value.id), + commas( + state.setTo(value.state) + ) + ) sql.update.run } def findLanguage(cid: Ident): ConnectionIO[Option[Language]] = - selectSimple(List(language), table, id is cid).query[Option[Language]].unique + selectSimple(List(language), table, id.is(cid)).query[Option[Language]].unique def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] = - updateRow(table, id is cid, language setTo lang).update.run + updateRow(table, id.is(cid), language.setTo(lang)).update.run def findById(cid: Ident): ConnectionIO[Option[RCollective]] = { - val sql = selectSimple(all, table, id is cid) + val sql = selectSimple(all, table, id.is(cid)) sql.query[RCollective].option } def existsById(cid: Ident): ConnectionIO[Boolean] = { - val sql = selectCount(id, table, id is cid) + val sql = selectCount(id, table, id.is(cid)) sql.query[Int].unique.map(_ > 0) } diff --git a/modules/store/src/main/scala/docspell/store/records/RContact.scala b/modules/store/src/main/scala/docspell/store/records/RContact.scala index ecb09d95..0df3459a 100644 --- a/modules/store/src/main/scala/docspell/store/records/RContact.scala +++ b/modules/store/src/main/scala/docspell/store/records/RContact.scala @@ -6,14 +6,13 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class RContact( - contactId: Ident - , value: String - , kind: ContactKind - , personId: Option[Ident] - , orgId: Option[Ident] - , created: Timestamp) { - -} + contactId: Ident, + value: String, + kind: ContactKind, + personId: Option[Ident], + orgId: Option[Ident], + created: Timestamp +) {} object RContact { @@ -21,53 +20,60 @@ object RContact { object Columns { val contactId = Column("contactid") - val value = Column("value") - val kind = Column("kind") - val personId = Column("pid") - val orgId = Column("oid") - val created = Column("created") - val all = List(contactId, value,kind, personId, orgId, created) + val value = Column("value") + val kind = Column("kind") + val personId = Column("pid") + val orgId = Column("oid") + val created = Column("created") + val all = List(contactId, value, kind, personId, orgId, created) } import Columns._ def insert(v: RContact): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}") + val sql = insertRow( + table, + all, + fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}" + ) sql.update.run } def update(v: RContact): ConnectionIO[Int] = { - val sql = updateRow(table, contactId is v.contactId, commas( - value setTo v.value, - kind setTo v.kind, - personId setTo v.personId, - orgId setTo v.orgId - )) + val sql = updateRow( + table, + contactId.is(v.contactId), + commas( + value.setTo(v.value), + kind.setTo(v.kind), + personId.setTo(v.personId), + orgId.setTo(v.orgId) + ) + ) sql.update.run } def delete(v: RContact): ConnectionIO[Int] = - deleteFrom(table, contactId is v.contactId).update.run + deleteFrom(table, contactId.is(v.contactId)).update.run def deleteOrg(oid: Ident): ConnectionIO[Int] = - deleteFrom(table, orgId is oid).update.run + deleteFrom(table, orgId.is(oid)).update.run def deletePerson(pid: Ident): ConnectionIO[Int] = - deleteFrom(table, personId is pid).update.run + deleteFrom(table, personId.is(pid)).update.run def findById(id: Ident): ConnectionIO[Option[RContact]] = { - val sql = selectSimple(all, table, contactId is id) + val sql = selectSimple(all, table, contactId.is(id)) sql.query[RContact].option } def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = { - val sql = selectSimple(all, table, personId is pid) + val sql = selectSimple(all, table, personId.is(pid)) sql.query[RContact].to[Vector] } def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = { - val sql = selectSimple(all, table, orgId is oid) + val sql = selectSimple(all, table, orgId.is(oid)) sql.query[RContact].to[Vector] } } diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala index d4384f5e..964bec4f 100644 --- a/modules/store/src/main/scala/docspell/store/records/REquipment.scala +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -5,61 +5,58 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class REquipment( - eid: Ident - , cid: Ident - , name: String - , created: Timestamp) { - -} +case class REquipment(eid: Ident, cid: Ident, name: String, created: Timestamp) {} object REquipment { val table = fr"equipment" object Columns { - val eid = Column("eid") - val cid = Column("cid") - val name = Column("name") + val eid = Column("eid") + val cid = Column("cid") + val name = Column("name") val created = Column("created") - val all = List(eid,cid,name,created) + val all = List(eid, cid, name, created) } import Columns._ def insert(v: REquipment): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.eid},${v.cid},${v.name},${v.created}") + val sql = insertRow(table, all, fr"${v.eid},${v.cid},${v.name},${v.created}") sql.update.run } def update(v: REquipment): ConnectionIO[Int] = { - val sql = updateRow(table, and(eid is v.eid, cid is v.cid), commas( - cid setTo v.cid, - name setTo v.name - )) + val sql = updateRow( + table, + and(eid.is(v.eid), cid.is(v.cid)), + commas( + cid.setTo(v.cid), + name.setTo(v.name) + ) + ) sql.update.run } def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = { - val sql = selectCount(eid, table, and(cid is coll, name is ename)) + val sql = selectCount(eid, table, and(cid.is(coll), name.is(ename))) sql.query[Int].unique.map(_ > 0) } def findById(id: Ident): ConnectionIO[Option[REquipment]] = { - val sql = selectSimple(all, table, eid is id) + val sql = selectSimple(all, table, eid.is(id)) sql.query[REquipment].option } def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[REquipment].to[Vector] } def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = - selectSimple(List(eid, name), table, and(cid is coll, - name.lowerLike(equipName))). - query[IdRef].to[Vector] + selectSimple(List(eid, name), table, and(cid.is(coll), name.lowerLike(equipName))) + .query[IdRef] + .to[Vector] def delete(id: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(eid is id, cid is coll)).update.run + deleteFrom(table, and(eid.is(id), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala index be7e49fe..98d48a51 100644 --- a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala +++ b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala @@ -8,43 +8,39 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class RInvitation(id: Ident, created: Timestamp) { - -} +case class RInvitation(id: Ident, created: Timestamp) {} object RInvitation { val table = fr"invitation" object Columns { - val id = Column("id") + val id = Column("id") val created = Column("created") - val all = List(id, created) + val all = List(id, created) } import Columns._ def generate[F[_]: Sync]: F[RInvitation] = for { - c <- Timestamp.current[F] - i <- Ident.randomId[F] + c <- Timestamp.current[F] + i <- Ident.randomId[F] } yield RInvitation(i, c) def insert(v: RInvitation): ConnectionIO[Int] = insertRow(table, all, fr"${v.id},${v.created}").update.run def insertNew: ConnectionIO[RInvitation] = - generate[ConnectionIO]. - flatMap(v => insert(v).map(_ => v)) + generate[ConnectionIO].flatMap(v => insert(v).map(_ => v)) def findById(invite: Ident): ConnectionIO[Option[RInvitation]] = - selectSimple(all, table, id is invite).query[RInvitation].option + selectSimple(all, table, id.is(invite)).query[RInvitation].option def delete(invite: Ident): ConnectionIO[Int] = - deleteFrom(table, id is invite).update.run + deleteFrom(table, id.is(invite)).update.run def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = { - val get = selectCount(id, table, and(id is invite, created isGt minCreated)). - query[Int].unique + val get = selectCount(id, table, and(id.is(invite), created.isGt(minCreated))).query[Int].unique for { inv <- get _ <- delete(invite) diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 99bd2d87..7d9dafda 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -8,156 +8,258 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class RItem( id: Ident - , cid: Ident - , name: String - , itemDate: Option[Timestamp] - , source: String - , direction: Direction - , state: ItemState - , corrOrg: Option[Ident] - , corrPerson: Option[Ident] - , concPerson: Option[Ident] - , concEquipment: Option[Ident] - , inReplyTo: Option[Ident] - , dueDate: Option[Timestamp] - , created: Timestamp - , updated: Timestamp - , notes: Option[String]) { - -} +case class RItem( + id: Ident, + cid: Ident, + name: String, + itemDate: Option[Timestamp], + source: String, + direction: Direction, + state: ItemState, + corrOrg: Option[Ident], + corrPerson: Option[Ident], + concPerson: Option[Ident], + concEquipment: Option[Ident], + inReplyTo: Option[Ident], + dueDate: Option[Timestamp], + created: Timestamp, + updated: Timestamp, + notes: Option[String] +) {} object RItem { - def newItem[F[_]: Sync](cid: Ident, name: String, source: String, direction: Direction, state: ItemState): F[RItem] = + def newItem[F[_]: Sync]( + cid: Ident, + name: String, + source: String, + direction: Direction, + state: ItemState + ): F[RItem] = for { now <- Timestamp.current[F] id <- Ident.randomId[F] - } yield RItem(id, cid, name, None, source, direction, state, None, None, None, None, None, None, now, now, None) + } yield RItem( + id, + cid, + name, + None, + source, + direction, + state, + None, + None, + None, + None, + None, + None, + now, + now, + None + ) val table = fr"item" object Columns { - val id = Column("itemid") - val cid = Column("cid") - val name = Column("name") - val itemDate = Column("itemdate") - val source = Column("source") - val incoming = Column("incoming") - val state = Column("state") - val corrOrg = Column("corrorg") - val corrPerson = Column("corrperson") - val concPerson = Column("concperson") + val id = Column("itemid") + val cid = Column("cid") + val name = Column("name") + val itemDate = Column("itemdate") + val source = Column("source") + val incoming = Column("incoming") + val state = Column("state") + val corrOrg = Column("corrorg") + val corrPerson = Column("corrperson") + val concPerson = Column("concperson") val concEquipment = Column("concequipment") - val inReplyTo = Column("inreplyto") - val dueDate = Column("duedate") - val created = Column("created") - val updated = Column("updated") - val notes = Column("notes") - val all = List(id, cid, name, itemDate, source, incoming, state, corrOrg, - corrPerson, concPerson, concEquipment, inReplyTo, dueDate, created, updated, notes) + val inReplyTo = Column("inreplyto") + val dueDate = Column("duedate") + val created = Column("created") + val updated = Column("updated") + val notes = Column("notes") + val all = List( + id, + cid, + name, + itemDate, + source, + incoming, + state, + corrOrg, + corrPerson, + concPerson, + concEquipment, + inReplyTo, + dueDate, + created, + updated, + notes + ) } import Columns._ def insert(v: RItem): ConnectionIO[Int] = - insertRow(table, all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++ - fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++ - fr"${v.created},${v.updated},${v.notes}").update.run + insertRow( + table, + all, + fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++ + fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++ + fr"${v.created},${v.updated},${v.notes}" + ).update.run def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(cid), table, id is itemId).query[Ident].option + selectSimple(List(cid), table, id.is(itemId)).query[Ident].option def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, id is itemId, commas(state setTo itemState, updated setTo t)).update.run + n <- updateRow(table, id.is(itemId), commas(state.setTo(itemState), updated.setTo(t))).update.run } yield n - def updateStateForCollective(itemId: Ident, itemState: ItemState, coll: Ident): ConnectionIO[Int] = + def updateStateForCollective( + itemId: Ident, + itemState: ItemState, + coll: Ident + ): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(state setTo itemState, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(state.setTo(itemState), updated.setTo(t)) + ).update.run } yield n def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(incoming setTo dir, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(incoming.setTo(dir), updated.setTo(t)) + ).update.run } yield n - def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(corrOrg setTo org, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(corrOrg.setTo(org), updated.setTo(t)) + ).update.run } yield n def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(cid is coll, corrOrg is Some(currentOrg)), commas(corrOrg setTo(None: Option[Ident]), updated setTo t)).update.run + n <- updateRow( + table, + and(cid.is(coll), corrOrg.is(Some(currentOrg))), + commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t)) + ).update.run } yield n def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(corrPerson setTo person, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(corrPerson.setTo(person), updated.setTo(t)) + ).update.run } yield n def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(cid is coll, corrPerson is Some(currentPerson)), commas(corrPerson setTo(None: Option[Ident]), updated setTo t)).update.run + n <- updateRow( + table, + and(cid.is(coll), corrPerson.is(Some(currentPerson))), + commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t)) + ).update.run } yield n def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(concPerson setTo person, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(concPerson.setTo(person), updated.setTo(t)) + ).update.run } yield n def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(cid is coll, concPerson is Some(currentPerson)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run + n <- updateRow( + table, + and(cid.is(coll), concPerson.is(Some(currentPerson))), + commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t)) + ).update.run } yield n def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(concEquipment setTo equip, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(concEquipment.setTo(equip), updated.setTo(t)) + ).update.run } yield n def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(cid is coll, concEquipment is Some(currentEquip)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run + n <- updateRow( + table, + and(cid.is(coll), concEquipment.is(Some(currentEquip))), + commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t)) + ).update.run } yield n def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(notes setTo text, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(notes.setTo(text), updated.setTo(t)) + ).update.run } yield n def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(name setTo itemName, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(name.setTo(itemName), updated.setTo(t)) + ).update.run } yield n def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(itemDate setTo date, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(itemDate.setTo(date), updated.setTo(t)) + ).update.run } yield n def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = for { t <- currentTime - n <- updateRow(table, and(id is itemId, cid is coll), commas(dueDate setTo date, updated setTo t)).update.run + n <- updateRow( + table, + and(id.is(itemId), cid.is(coll)), + commas(dueDate.setTo(date), updated.setTo(t)) + ).update.run } yield n def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(id is itemId, cid is coll)).update.run + deleteFrom(table, and(id.is(itemId), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RJob.scala b/modules/store/src/main/scala/docspell/store/records/RJob.scala index 07d5bcad..d5a9c039 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJob.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJob.scala @@ -8,21 +8,23 @@ import docspell.store.impl.Column import docspell.store.impl.Implicits._ import io.circe.Encoder -case class RJob(id: Ident - , task: Ident - , group: Ident - , args: String - , subject: String - , submitted: Timestamp - , submitter: Ident - , priority: Priority - , state: JobState - , retries: Int - , progress: Int - , tracker: Option[Ident] - , worker: Option[Ident] - , started: Option[Timestamp] - , finished: Option[Timestamp]) { +case class RJob( + id: Ident, + task: Ident, + group: Ident, + args: String, + subject: String, + submitted: Timestamp, + submitter: Ident, + priority: Priority, + state: JobState, + retries: Int, + progress: Int, + tracker: Option[Ident], + worker: Option[Ident], + started: Option[Timestamp], + finished: Option[Timestamp] +) { def info: String = s"${id.id.substring(0, 9)}.../${group.id}/${task.id}/$priority" @@ -30,136 +32,208 @@ case class RJob(id: Ident object RJob { - def newJob[A](id: Ident - , task: Ident - , group: Ident - , args: A - , subject: String - , submitted: Timestamp - , submitter: Ident - , priority: Priority - , tracker: Option[Ident])(implicit E: Encoder[A]): RJob = - RJob(id, task, group, E(args).noSpaces, subject, submitted, submitter, priority, JobState.Waiting, 0, 0, tracker, None, None, None) + def newJob[A]( + id: Ident, + task: Ident, + group: Ident, + args: A, + subject: String, + submitted: Timestamp, + submitter: Ident, + priority: Priority, + tracker: Option[Ident] + )(implicit E: Encoder[A]): RJob = + RJob( + id, + task, + group, + E(args).noSpaces, + subject, + submitted, + submitter, + priority, + JobState.Waiting, + 0, + 0, + tracker, + None, + None, + None + ) val table = fr"job" object Columns { - val id = Column("jid") - val task = Column("task") - val group = Column("group_") - val args = Column("args") - val subject = Column("subject") - val submitted = Column("submitted") - val submitter = Column("submitter") - val priority = Column("priority") - val state = Column("state") - val retries = Column("retries") - val progress = Column("progress") - val tracker = Column("tracker") - val worker = Column("worker") - val started = Column("started") + val id = Column("jid") + val task = Column("task") + val group = Column("group_") + val args = Column("args") + val subject = Column("subject") + val submitted = Column("submitted") + val submitter = Column("submitter") + val priority = Column("priority") + val state = Column("state") + val retries = Column("retries") + val progress = Column("progress") + val tracker = Column("tracker") + val worker = Column("worker") + val started = Column("started") val startedmillis = Column("startedmillis") - val finished = Column("finished") - val all = List(id,task,group,args,subject,submitted,submitter,priority,state,retries,progress,tracker,worker,started,finished) + val finished = Column("finished") + val all = List( + id, + task, + group, + args, + subject, + submitted, + submitter, + priority, + state, + retries, + progress, + tracker, + worker, + started, + finished + ) } import Columns._ def insert(v: RJob): ConnectionIO[Int] = { val smillis = v.started.map(_.toMillis) - val sql = insertRow(table, all ++ List(startedmillis), - fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis") + val sql = insertRow( + table, + all ++ List(startedmillis), + fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis" + ) sql.update.run } - def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] = { + def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] = if (ids.isEmpty) Sync[ConnectionIO].pure(Vector.empty[RJob]) - else selectSimple(all, table, id isOneOf ids).query[RJob].to[Vector] - } + else selectSimple(all, table, id.isOneOf(ids)).query[RJob].to[Vector] def findByIdAndGroup(jobId: Ident, jobGroup: Ident): ConnectionIO[Option[RJob]] = - selectSimple(all, table, and(id is jobId, group is jobGroup)).query[RJob].option + selectSimple(all, table, and(id.is(jobId), group.is(jobGroup))).query[RJob].option def setRunningToWaiting(workerId: Ident): ConnectionIO[Int] = { val states: Seq[JobState] = List(JobState.Running, JobState.Scheduled) - updateRow(table, and(worker is workerId, state isOneOf states), - state setTo (JobState.Waiting: JobState)).update.run + updateRow( + table, + and(worker.is(workerId), state.isOneOf(states)), + state.setTo(JobState.Waiting: JobState) + ).update.run } def incrementRetries(jobid: Ident): ConnectionIO[Int] = - updateRow(table, and(id is jobid, state is (JobState.Stuck: JobState)), - retries.f ++ fr"=" ++ retries.f ++ fr"+ 1").update.run + updateRow( + table, + and(id.is(jobid), state.is(JobState.Stuck: JobState)), + retries.f ++ fr"=" ++ retries.f ++ fr"+ 1" + ).update.run def setRunning(jobId: Ident, workerId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Running: JobState), - started setTo now, - startedmillis setTo now.toMillis, - worker setTo workerId - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Running: JobState), + started.setTo(now), + startedmillis.setTo(now.toMillis), + worker.setTo(workerId) + ) + ).update.run def setWaiting(jobId: Ident): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Waiting: JobState), - started setTo (None: Option[Timestamp]), - startedmillis setTo (None: Option[Long]), - finished setTo (None: Option[Timestamp]) - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Waiting: JobState), + started.setTo(None: Option[Timestamp]), + startedmillis.setTo(None: Option[Long]), + finished.setTo(None: Option[Timestamp]) + ) + ).update.run - def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] = { + def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] = for { - _ <- incrementRetries(jobId) - n <- updateRow(table, and(id is jobId, or(worker.isNull, worker is workerId), state isOneOf Seq[JobState](JobState.Waiting, JobState.Stuck)), commas( - state setTo (JobState.Scheduled: JobState), - worker setTo workerId - )).update.run + _ <- incrementRetries(jobId) + n <- updateRow( + table, + and( + id.is(jobId), + or(worker.isNull, worker.is(workerId)), + state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck)) + ), + commas( + state.setTo(JobState.Scheduled: JobState), + worker.setTo(workerId) + ) + ).update.run } yield n - } def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Success: JobState), - finished setTo now - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Success: JobState), + finished.setTo(now) + ) + ).update.run def setStuck(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Stuck: JobState), - finished setTo now - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Stuck: JobState), + finished.setTo(now) + ) + ).update.run def setFailed(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Failed: JobState), - finished setTo now - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Failed: JobState), + finished.setTo(now) + ) + ).update.run def setCancelled(jobId: Ident, now: Timestamp): ConnectionIO[Int] = - updateRow(table, id is jobId, commas( - state setTo (JobState.Cancelled: JobState), - finished setTo now - )).update.run + updateRow( + table, + id.is(jobId), + commas( + state.setTo(JobState.Cancelled: JobState), + finished.setTo(now) + ) + ).update.run def getRetries(jobId: Ident): ConnectionIO[Option[Int]] = - selectSimple(List(retries), table, id is jobId).query[Int].option + selectSimple(List(retries), table, id.is(jobId)).query[Int].option def setProgress(jobId: Ident, perc: Int): ConnectionIO[Int] = - updateRow(table, id is jobId, progress setTo perc).update.run + updateRow(table, id.is(jobId), progress.setTo(perc)).update.run def selectWaiting: ConnectionIO[Option[RJob]] = { - val sql = selectSimple(all, table, state is (JobState.Waiting: JobState)) + val sql = selectSimple(all, table, state.is(JobState.Waiting: JobState)) sql.query[RJob].to[Vector].map(_.headOption) } def selectGroupInState(states: Seq[JobState]): ConnectionIO[Vector[Ident]] = { - val sql = selectDistinct(List(group), table, state isOneOf states) ++ orderBy(group.f) + val sql = selectDistinct(List(group), table, state.isOneOf(states)) ++ orderBy(group.f) sql.query[Ident].to[Vector] } - def delete(jobId: Ident): ConnectionIO[Int] = { + def delete(jobId: Ident): ConnectionIO[Int] = for { n0 <- RJobLog.deleteAll(jobId) - n1 <- deleteFrom(table, id is jobId).update.run + n1 <- deleteFrom(table, id.is(jobId)).update.run } yield n0 + n1 - } } diff --git a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala index a06bb16e..8210da33 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala @@ -7,18 +7,16 @@ import docspell.common._ import docspell.store.impl.Column import docspell.store.impl.Implicits._ -case class RJobGroupUse(groupId: Ident, workerId: Ident) { - -} +case class RJobGroupUse(groupId: Ident, workerId: Ident) {} object RJobGroupUse { val table = fr"jobgroupuse" object Columns { - val group = Column("groupid") + val group = Column("groupid") val worker = Column("workerid") - val all = List(group, worker) + val all = List(group, worker) } import Columns._ @@ -26,12 +24,11 @@ object RJobGroupUse { insertRow(table, all, fr"${v.groupId},${v.workerId}").update.run def updateGroup(v: RJobGroupUse): ConnectionIO[Int] = - updateRow(table, worker is v.workerId, group setTo v.groupId).update.run + updateRow(table, worker.is(v.workerId), group.setTo(v.groupId)).update.run - def setGroup(v: RJobGroupUse): ConnectionIO[Int] = { + def setGroup(v: RJobGroupUse): ConnectionIO[Int] = updateGroup(v).flatMap(n => if (n > 0) n.pure[ConnectionIO] else insert(v)) - } def findGroup(workerId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(group), table, worker is workerId).query[Ident].option + selectSimple(List(group), table, worker.is(workerId)).query[Ident].option } diff --git a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala index e3aad8c8..0f729626 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala @@ -6,25 +6,19 @@ import docspell.common._ import docspell.store.impl.Column import docspell.store.impl.Implicits._ -case class RJobLog( id: Ident - , jobId: Ident - , level: LogLevel - , created: Timestamp - , message: String) { - -} +case class RJobLog(id: Ident, jobId: Ident, level: LogLevel, created: Timestamp, message: String) {} object RJobLog { val table = fr"joblog" object Columns { - val id = Column("id") - val jobId = Column("jid") - val level = Column("level") + val id = Column("id") + val jobId = Column("jid") + val level = Column("level") val created = Column("created") val message = Column("message") - val all = List(id, jobId, level, created, message) + val all = List(id, jobId, level, created, message) } import Columns._ @@ -32,8 +26,8 @@ object RJobLog { insertRow(table, all, fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}").update.run def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] = - (selectSimple(all, table, jobId is id) ++ orderBy(created.asc)).query[RJobLog].to[Vector] + (selectSimple(all, table, jobId.is(id)) ++ orderBy(created.asc)).query[RJobLog].to[Vector] def deleteAll(job: Ident): ConnectionIO[Int] = - deleteFrom(table, jobId is job).update.run + deleteFrom(table, jobId.is(job)).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala index ec42265c..51c6e51e 100644 --- a/modules/store/src/main/scala/docspell/store/records/RNode.scala +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -8,10 +8,13 @@ import docspell.common._ import docspell.store.impl.Column import docspell.store.impl.Implicits._ - -case class RNode(id: Ident, nodeType: NodeType, url: LenientUri, updated: Timestamp, created: Timestamp) { - -} +case class RNode( + id: Ident, + nodeType: NodeType, + url: LenientUri, + updated: Timestamp, + created: Timestamp +) {} object RNode { @@ -21,12 +24,12 @@ object RNode { val table = fr"node" object Columns { - val id = Column("id") + val id = Column("id") val nodeType = Column("type") - val url = Column("url") - val updated = Column("updated") - val created = Column("created") - val all = List(id,nodeType,url,updated,created) + val url = Column("url") + val updated = Column("updated") + val created = Column("created") + val all = List(id, nodeType, url, updated, created) } import Columns._ @@ -34,24 +37,28 @@ object RNode { insertRow(table, all, fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}").update.run def update(v: RNode): ConnectionIO[Int] = - updateRow(table, id is v.id, commas( - nodeType setTo v.nodeType, - url setTo v.url, - updated setTo v.updated - )).update.run + updateRow( + table, + id.is(v.id), + commas( + nodeType.setTo(v.nodeType), + url.setTo(v.url), + updated.setTo(v.updated) + ) + ).update.run def set(v: RNode): ConnectionIO[Int] = for { - n <- update(v) - k <- if (n == 0) insert(v) else 0.pure[ConnectionIO] + n <- update(v) + k <- if (n == 0) insert(v) else 0.pure[ConnectionIO] } yield n + k def delete(appId: Ident): ConnectionIO[Int] = - (fr"DELETE FROM" ++ table ++ where(id is appId)).update.run + (fr"DELETE FROM" ++ table ++ where(id.is(appId))).update.run def findAll(nt: NodeType): ConnectionIO[Vector[RNode]] = - selectSimple(all, table, nodeType is nt).query[RNode].to[Vector] + selectSimple(all, table, nodeType.is(nt)).query[RNode].to[Vector] def findById(nodeId: Ident): ConnectionIO[Option[RNode]] = - selectSimple(all, table, id is nodeId).query[RNode].option + selectSimple(all, table, id.is(nodeId)).query[RNode].option } diff --git a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala index d09127fb..d4b333ee 100644 --- a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala +++ b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala @@ -8,96 +8,108 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class ROrganization( - oid: Ident - , cid: Ident - , name: String - , street: String - , zip: String - , city: String - , country: String - , notes: Option[String] - , created: Timestamp) { - -} + oid: Ident, + cid: Ident, + name: String, + street: String, + zip: String, + city: String, + country: String, + notes: Option[String], + created: Timestamp +) {} object ROrganization { val table = fr"organization" object Columns { - val oid = Column("oid") - val cid = Column("cid") - val name = Column("name") - val street = Column("street") - val zip = Column("zip") - val city = Column("city") + val oid = Column("oid") + val cid = Column("cid") + val name = Column("name") + val street = Column("street") + val zip = Column("zip") + val city = Column("city") val country = Column("country") - val notes = Column("notes") + val notes = Column("notes") val created = Column("created") - val all = List(oid, cid, name, street, zip, city, country, notes, created) + val all = List(oid, cid, name, street, zip, city, country, notes, created) } import Columns._ def insert(v: ROrganization): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created}") + val sql = insertRow( + table, + all, + fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created}" + ) sql.update.run } def update(v: ROrganization): ConnectionIO[Int] = { - val sql = updateRow(table, and(oid is v.oid, cid is v.cid), commas( - cid setTo v.cid, - name setTo v.name, - street setTo v.street, - zip setTo v.zip, - city setTo v.city, - country setTo v.country, - notes setTo v.notes - )) + val sql = updateRow( + table, + and(oid.is(v.oid), cid.is(v.cid)), + commas( + cid.setTo(v.cid), + name.setTo(v.name), + street.setTo(v.street), + zip.setTo(v.zip), + city.setTo(v.city), + country.setTo(v.country), + notes.setTo(v.notes) + ) + ) sql.update.run } def existsByName(coll: Ident, oname: String): ConnectionIO[Boolean] = - selectCount(oid, table, and(cid is coll, name is oname)).query[Int].unique.map(_ > 0) + selectCount(oid, table, and(cid.is(coll), name.is(oname))).query[Int].unique.map(_ > 0) def findById(id: Ident): ConnectionIO[Option[ROrganization]] = { - val sql = selectSimple(all, table, cid is id) + val sql = selectSimple(all, table, cid.is(id)) sql.query[ROrganization].option } def find(coll: Ident, orgName: String): ConnectionIO[Option[ROrganization]] = { - val sql = selectSimple(all, table, and(cid is coll, name is orgName)) + val sql = selectSimple(all, table, and(cid.is(coll), name.is(orgName))) sql.query[ROrganization].option } def findLike(coll: Ident, orgName: String): ConnectionIO[Vector[IdRef]] = - selectSimple(List(oid, name), table, and(cid is coll, - name.lowerLike(orgName))). - query[IdRef].to[Vector] + selectSimple(List(oid, name), table, and(cid.is(coll), name.lowerLike(orgName))) + .query[IdRef] + .to[Vector] - def findLike(coll: Ident, contactKind: ContactKind, value: String): ConnectionIO[Vector[IdRef]] = { + def findLike( + coll: Ident, + contactKind: ContactKind, + value: String + ): ConnectionIO[Vector[IdRef]] = { val CC = RContact.Columns val q = fr"SELECT DISTINCT" ++ commas(oid.prefix("o").f, name.prefix("o").f) ++ fr"FROM" ++ table ++ fr"o" ++ fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.orgId.prefix("c").is(oid.prefix("o")) ++ - fr"WHERE" ++ and(cid.prefix("o").is(coll) - , CC.kind.prefix("c") is contactKind - , CC.value.prefix("c").lowerLike(value)) + fr"WHERE" ++ and( + cid.prefix("o").is(coll), + CC.kind.prefix("c").is(contactKind), + CC.value.prefix("c").lowerLike(value) + ) q.query[IdRef].to[Vector] } def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, ROrganization] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[ROrganization].stream } def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = { - val sql = selectSimple(List(oid, name), table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(List(oid, name), table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[IdRef].to[Vector] } def delete(id: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(oid is id, cid is coll)).update.run + deleteFrom(table, and(oid.is(id), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RPerson.scala b/modules/store/src/main/scala/docspell/store/records/RPerson.scala index 3941d441..184367b2 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPerson.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPerson.scala @@ -8,101 +8,119 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class RPerson( - pid: Ident - , cid: Ident - , name: String - , street: String - , zip: String - , city: String - , country: String - , notes: Option[String] - , concerning: Boolean - , created: Timestamp) { - -} + pid: Ident, + cid: Ident, + name: String, + street: String, + zip: String, + city: String, + country: String, + notes: Option[String], + concerning: Boolean, + created: Timestamp +) {} object RPerson { val table = fr"person" object Columns { - val pid = Column("pid") - val cid = Column("cid") - val name = Column("name") - val street = Column("street") - val zip = Column("zip") - val city = Column("city") - val country = Column("country") - val notes = Column("notes") + val pid = Column("pid") + val cid = Column("cid") + val name = Column("name") + val street = Column("street") + val zip = Column("zip") + val city = Column("city") + val country = Column("country") + val notes = Column("notes") val concerning = Column("concerning") - val created = Column("created") - val all = List(pid, cid, name, street, zip, city, country, notes, concerning, created) + val created = Column("created") + val all = List(pid, cid, name, street, zip, city, country, notes, concerning, created) } import Columns._ def insert(v: RPerson): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created}") + val sql = insertRow( + table, + all, + fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created}" + ) sql.update.run } def update(v: RPerson): ConnectionIO[Int] = { - val sql = updateRow(table, and(pid is v.pid, cid is v.cid), commas( - cid setTo v.cid, - name setTo v.name, - street setTo v.street, - zip setTo v.zip, - city setTo v.city, - country setTo v.country, - concerning setTo v.concerning, - notes setTo v.notes - )) + val sql = updateRow( + table, + and(pid.is(v.pid), cid.is(v.cid)), + commas( + cid.setTo(v.cid), + name.setTo(v.name), + street.setTo(v.street), + zip.setTo(v.zip), + city.setTo(v.city), + country.setTo(v.country), + concerning.setTo(v.concerning), + notes.setTo(v.notes) + ) + ) sql.update.run } def existsByName(coll: Ident, pname: String): ConnectionIO[Boolean] = - selectCount(pid, table, and(cid is coll, name is pname)).query[Int].unique.map(_ > 0) + selectCount(pid, table, and(cid.is(coll), name.is(pname))).query[Int].unique.map(_ > 0) def findById(id: Ident): ConnectionIO[Option[RPerson]] = { - val sql = selectSimple(all, table, cid is id) + val sql = selectSimple(all, table, cid.is(id)) sql.query[RPerson].option } def find(coll: Ident, personName: String): ConnectionIO[Option[RPerson]] = { - val sql = selectSimple(all, table, and(cid is coll, name is personName)) + val sql = selectSimple(all, table, and(cid.is(coll), name.is(personName))) sql.query[RPerson].option } - def findLike(coll: Ident, personName: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] = - selectSimple(List(pid, name), table, and(cid is coll, - concerning is concerningOnly, - name.lowerLike(personName))). - query[IdRef].to[Vector] + def findLike( + coll: Ident, + personName: String, + concerningOnly: Boolean + ): ConnectionIO[Vector[IdRef]] = + selectSimple( + List(pid, name), + table, + and(cid.is(coll), concerning.is(concerningOnly), name.lowerLike(personName)) + ).query[IdRef].to[Vector] - def findLike(coll: Ident, contactKind: ContactKind, value: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] = { + def findLike( + coll: Ident, + contactKind: ContactKind, + value: String, + concerningOnly: Boolean + ): ConnectionIO[Vector[IdRef]] = { val CC = RContact.Columns val q = fr"SELECT DISTINCT" ++ commas(pid.prefix("p").f, name.prefix("p").f) ++ fr"FROM" ++ table ++ fr"p" ++ fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.personId.prefix("c").is(pid.prefix("p")) ++ - fr"WHERE" ++ and(cid.prefix("p").is(coll) - , CC.kind.prefix("c") is contactKind - , concerning.prefix("p") is concerningOnly - , CC.value.prefix("c").lowerLike(value)) + fr"WHERE" ++ and( + cid.prefix("p").is(coll), + CC.kind.prefix("c").is(contactKind), + concerning.prefix("p").is(concerningOnly), + CC.value.prefix("c").lowerLike(value) + ) q.query[IdRef].to[Vector] } def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, RPerson] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[RPerson].stream } def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = { - val sql = selectSimple(List(pid, name), table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(List(pid, name), table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[IdRef].to[Vector] } def delete(personId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(pid is personId, cid is coll)).update.run + deleteFrom(table, and(pid.is(personId), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala index c4a99421..f66c9e13 100644 --- a/modules/store/src/main/scala/docspell/store/records/RSource.scala +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -7,16 +7,15 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class RSource( - sid: Ident - , cid: Ident - , abbrev: String - , description: Option[String] - , counter: Int - , enabled: Boolean - , priority: Priority - , created: Timestamp) { - -} + sid: Ident, + cid: Ident, + abbrev: String, + description: Option[String], + counter: Int, + enabled: Boolean, + priority: Priority, + created: Timestamp +) {} object RSource { @@ -24,64 +23,74 @@ object RSource { object Columns { - val sid = Column("sid") - val cid = Column("cid") - val abbrev = Column("abbrev") + val sid = Column("sid") + val cid = Column("cid") + val abbrev = Column("abbrev") val description = Column("description") - val counter = Column("counter") - val enabled = Column("enabled") - val priority = Column("priority") - val created = Column("created") + val counter = Column("counter") + val enabled = Column("enabled") + val priority = Column("priority") + val created = Column("created") - val all = List(sid,cid,abbrev,description,counter,enabled,priority,created) + val all = List(sid, cid, abbrev, description, counter, enabled, priority, created) } import Columns._ def insert(v: RSource): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}") + val sql = insertRow( + table, + all, + fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}" + ) sql.update.run } def updateNoCounter(v: RSource): ConnectionIO[Int] = { - val sql = updateRow(table, and(sid is v.sid, cid is v.cid), commas( - cid setTo v.cid, - abbrev setTo v.abbrev, - description setTo v.description, - enabled setTo v.enabled, - priority setTo v.priority - )) + val sql = updateRow( + table, + and(sid.is(v.sid), cid.is(v.cid)), + commas( + cid.setTo(v.cid), + abbrev.setTo(v.abbrev), + description.setTo(v.description), + enabled.setTo(v.enabled), + priority.setTo(v.priority) + ) + ) sql.update.run } def incrementCounter(source: String, coll: Ident): ConnectionIO[Int] = - updateRow(table, and(abbrev is source, cid is coll), counter.f ++ fr"=" ++ counter.f ++ fr"+ 1").update.run + updateRow( + table, + and(abbrev.is(source), cid.is(coll)), + counter.f ++ fr"=" ++ counter.f ++ fr"+ 1" + ).update.run def existsById(id: Ident): ConnectionIO[Boolean] = { - val sql = selectCount(sid, table, sid is id) + val sql = selectCount(sid, table, sid.is(id)) sql.query[Int].unique.map(_ > 0) } def existsByAbbrev(coll: Ident, abb: String): ConnectionIO[Boolean] = { - val sql = selectCount(sid, table, and(cid is coll, abbrev is abb)) + val sql = selectCount(sid, table, and(cid.is(coll), abbrev.is(abb))) sql.query[Int].unique.map(_ > 0) } - def find(id: Ident): ConnectionIO[Option[RSource]] = { - val sql = selectSimple(all, table, sid is id) + val sql = selectSimple(all, table, sid.is(id)) sql.query[RSource].option } def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] = - selectSimple(List(cid), table, sid is sourceId).query[Ident].option + selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RSource]] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[RSource].to[Vector] } def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(sid is sourceId, cid is coll)).update.run + deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RTag.scala b/modules/store/src/main/scala/docspell/store/records/RTag.scala index 246febf0..52ac5f2c 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTag.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala @@ -6,71 +6,82 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class RTag( - tagId: Ident - , collective: Ident - , name: String - , category: Option[String] - , created: Timestamp) { - -} + tagId: Ident, + collective: Ident, + name: String, + category: Option[String], + created: Timestamp +) {} object RTag { val table = fr"tag" object Columns { - val tid = Column("tid") - val cid = Column("cid") - val name = Column("name") + val tid = Column("tid") + val cid = Column("cid") + val name = Column("name") val category = Column("category") - val created = Column("created") - val all = List(tid,cid,name,category,created) + val created = Column("created") + val all = List(tid, cid, name, category, created) } import Columns._ def insert(v: RTag): ConnectionIO[Int] = { - val sql = insertRow(table, all, - fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}") + val sql = + insertRow(table, all, fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}") sql.update.run } def update(v: RTag): ConnectionIO[Int] = { - val sql = updateRow(table, and(tid is v.tagId, cid is v.collective), commas( - cid setTo v.collective, - name setTo v.name, - category setTo v.category - )) + val sql = updateRow( + table, + and(tid.is(v.tagId), cid.is(v.collective)), + commas( + cid.setTo(v.collective), + name.setTo(v.name), + category.setTo(v.category) + ) + ) sql.update.run } def findById(id: Ident): ConnectionIO[Option[RTag]] = { - val sql = selectSimple(all, table, tid is id) + val sql = selectSimple(all, table, tid.is(id)) sql.query[RTag].option } def findByIdAndCollective(id: Ident, coll: Ident): ConnectionIO[Option[RTag]] = { - val sql = selectSimple(all, table, and(tid is id, cid is coll)) + val sql = selectSimple(all, table, and(tid.is(id), cid.is(coll))) sql.query[RTag].option } def existsByName(tag: RTag): ConnectionIO[Boolean] = { - val sql = selectCount(tid, table, and(cid is tag.collective, name is tag.name, category is tag.category)) + val sql = selectCount( + tid, + table, + and(cid.is(tag.collective), name.is(tag.name), category.is(tag.category)) + ) sql.query[Int].unique.map(_ > 0) } def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RTag]] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[RTag].to[Vector] } def findByItem(itemId: Ident): ConnectionIO[Vector[RTag]] = { val rcol = all.map(_.prefix("t")) - (selectSimple(rcol, table ++ fr"t," ++ RTagItem.table ++ fr"i", and( - RTagItem.Columns.itemId.prefix("i") is itemId, - RTagItem.Columns.tagId.prefix("i").is(tid.prefix("t")) - )) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector] + (selectSimple( + rcol, + table ++ fr"t," ++ RTagItem.table ++ fr"i", + and( + RTagItem.Columns.itemId.prefix("i").is(itemId), + RTagItem.Columns.tagId.prefix("i").is(tid.prefix("t")) + ) + ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector] } def delete(tagId: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(tid is tagId, cid is coll)).update.run + deleteFrom(table, and(tid.is(tagId), cid.is(coll))).update.run } diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index 68c8f75e..e72f3351 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -7,9 +7,7 @@ import docspell.common._ import docspell.store.impl._ import docspell.store.impl.Implicits._ -case class RTagItem(tagItemId: Ident, itemId: Ident, tagId: Ident) { - -} +case class RTagItem(tagItemId: Ident, itemId: Ident, tagId: Ident) {} object RTagItem { @@ -17,9 +15,9 @@ object RTagItem { object Columns { val tagItemId = Column("tagitemid") - val itemId = Column("itemid") - val tagId = Column("tid") - val all = List(tagItemId, itemId, tagId) + val itemId = Column("itemid") + val tagId = Column("tid") + val all = List(tagItemId, itemId, tagId) } import Columns._ @@ -27,16 +25,17 @@ object RTagItem { insertRow(table, all, fr"${v.tagItemId},${v.itemId},${v.tagId}").update.run def deleteItemTags(item: Ident): ConnectionIO[Int] = - deleteFrom(table, itemId is item).update.run + deleteFrom(table, itemId.is(item)).update.run def deleteTag(tid: Ident): ConnectionIO[Int] = - deleteFrom(table, tagId is tid).update.run + deleteFrom(table, tagId.is(tid)).update.run - def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = { + def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = for { - tagValues <- tags.toList.traverse(id => Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))) - tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") - ins <- insertRows(table, all, tagFrag).update.run + tagValues <- tags.toList.traverse(id => + Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id)) + ) + tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") + ins <- insertRows(table, all, tagFrag).update.run } yield ins - } } diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala index bbfa3c38..5afc1be7 100644 --- a/modules/store/src/main/scala/docspell/store/records/RUser.scala +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -6,94 +6,97 @@ import docspell.store.impl._ import docspell.store.impl.Implicits._ case class RUser( - uid: Ident - , login: Ident - , cid: Ident - , password: Password - , state: UserState - , email: Option[String] - , loginCount: Int - , lastLogin: Option[Timestamp] - , created: Timestamp) { - -} + uid: Ident, + login: Ident, + cid: Ident, + password: Password, + state: UserState, + email: Option[String], + loginCount: Int, + lastLogin: Option[Timestamp], + created: Timestamp +) {} object RUser { val table = fr"user_" object Columns { - val uid = Column("uid") - val cid = Column("cid") - val login = Column("login") - val password = Column("password") - val state = Column("state") - val email = Column("email") + val uid = Column("uid") + val cid = Column("cid") + val login = Column("login") + val password = Column("password") + val state = Column("state") + val email = Column("email") val loginCount = Column("logincount") - val lastLogin = Column("lastlogin") - val created = Column("created") + val lastLogin = Column("lastlogin") + val created = Column("created") - val all = List(uid - ,login - ,cid - ,password - ,state - ,email - ,loginCount - ,lastLogin - ,created) + val all = List(uid, login, cid, password, state, email, loginCount, lastLogin, created) } import Columns._ def insert(v: RUser): ConnectionIO[Int] = { - val sql = insertRow(table, Columns.all, - fr"${v.uid},${v.login},${v.cid},${v.password},${v.state},${v.email},${v.loginCount},${v.lastLogin},${v.created}") + val sql = insertRow( + table, + Columns.all, + fr"${v.uid},${v.login},${v.cid},${v.password},${v.state},${v.email},${v.loginCount},${v.lastLogin},${v.created}" + ) sql.update.run } def update(v: RUser): ConnectionIO[Int] = { - val sql = updateRow(table, and(login is v.login, cid is v.cid), commas( - state setTo v.state, - email setTo v.email, - loginCount setTo v.loginCount, - lastLogin setTo v.lastLogin - )) + val sql = updateRow( + table, + and(login.is(v.login), cid.is(v.cid)), + commas( + state.setTo(v.state), + email.setTo(v.email), + loginCount.setTo(v.loginCount), + lastLogin.setTo(v.lastLogin) + ) + ) sql.update.run } - def exists(loginName: Ident): ConnectionIO[Boolean] = { - selectCount(uid, table, login is loginName).query[Int].unique.map(_ > 0) - } + def exists(loginName: Ident): ConnectionIO[Boolean] = + selectCount(uid, table, login.is(loginName)).query[Int].unique.map(_ > 0) def findByAccount(aid: AccountId): ConnectionIO[Option[RUser]] = { - val sql = selectSimple(all, table, and(cid is aid.collective, login is aid.user)) + val sql = selectSimple(all, table, and(cid.is(aid.collective), login.is(aid.user))) sql.query[RUser].option } def findById(userId: Ident): ConnectionIO[Option[RUser]] = { - val sql = selectSimple(all, table, uid is userId) + val sql = selectSimple(all, table, uid.is(userId)) sql.query[RUser].option } def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RUser]] = { - val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f) sql.query[RUser].to[Vector] } def updateLogin(accountId: AccountId): ConnectionIO[Int] = - currentTime.flatMap(t => updateRow(table - , and(cid is accountId.collective, login is accountId.user) - , commas( - loginCount.f ++ fr"=" ++ loginCount.f ++ fr"+ 1", - lastLogin setTo t - )).update.run) + currentTime.flatMap(t => + updateRow( + table, + and(cid.is(accountId.collective), login.is(accountId.user)), + commas( + loginCount.f ++ fr"=" ++ loginCount.f ++ fr"+ 1", + lastLogin.setTo(t) + ) + ).update.run + ) def updatePassword(accountId: AccountId, hashedPass: Password): ConnectionIO[Int] = - updateRow(table - , and(cid is accountId.collective, login is accountId.user) - , password setTo hashedPass).update.run + updateRow( + table, + and(cid.is(accountId.collective), login.is(accountId.user)), + password.setTo(hashedPass) + ).update.run def delete(user: Ident, coll: Ident): ConnectionIO[Int] = - deleteFrom(table, and(cid is coll, login is user)).update.run + deleteFrom(table, and(cid.is(coll), login.is(user))).update.run } diff --git a/modules/text/src/main/scala/docspell/text/contact/Domain.scala b/modules/text/src/main/scala/docspell/text/contact/Domain.scala index 5b62aba9..b9973392 100644 --- a/modules/text/src/main/scala/docspell/text/contact/Domain.scala +++ b/modules/text/src/main/scala/docspell/text/contact/Domain.scala @@ -16,24 +16,28 @@ case class Domain(labels: NonEmptyList[String], tld: String) { object Domain { def domainFromUri(uri: String): Either[String, Domain] = - LenientUri.parse(if (uri.contains("://")) uri else s"http://$uri"). - flatMap(uri => uri.authority.toRight("Uri has no authoriry part")). - flatMap(auth => parse(auth)) + LenientUri + .parse(if (uri.contains("://")) uri else s"http://$uri") + .flatMap(uri => uri.authority.toRight("Uri has no authoriry part")) + .flatMap(auth => parse(auth)) - def parse(str: String): Either[String, Domain] = { - Tld.findTld(str). - map(tld => (str.dropRight(tld.length), tld)). - map({ case (names, tld) => - names.split('.').toList match { - case Nil => Left(s"Not a domain: $str") - case segs if segs.forall(label => - label.trim.nonEmpty && label.forall(c => c.isLetter || c.isDigit || c == '-')) => - Right(Domain(NonEmptyList.fromListUnsafe(segs), tld)) - case _ => Left(s"Not a domain: $str") - } - }). - getOrElse(Left(s"Not a domain $str")) - } + def parse(str: String): Either[String, Domain] = + Tld + .findTld(str) + .map(tld => (str.dropRight(tld.length), tld)) + .map({ + case (names, tld) => + names.split('.').toList match { + case Nil => Left(s"Not a domain: $str") + case segs + if segs.forall(label => + label.trim.nonEmpty && label.forall(c => c.isLetter || c.isDigit || c == '-') + ) => + Right(Domain(NonEmptyList.fromListUnsafe(segs), tld)) + case _ => Left(s"Not a domain: $str") + } + }) + .getOrElse(Left(s"Not a domain $str")) def isDomain(str: String): Boolean = parse(str).isRight diff --git a/modules/text/src/main/scala/docspell/text/ocr/File.scala b/modules/text/src/main/scala/docspell/text/ocr/File.scala index 91bc5dcf..31d05d3b 100644 --- a/modules/text/src/main/scala/docspell/text/ocr/File.scala +++ b/modules/text/src/main/scala/docspell/text/ocr/File.scala @@ -20,20 +20,23 @@ object File { def deleteDirectory[F[_]: Sync](dir: Path): F[Int] = Sync[F].delay { val count = new AtomicInteger(0) - Files.walkFileTree(dir, new SimpleFileVisitor[Path]() { - override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { - Files.deleteIfExists(file) - count.incrementAndGet() - FileVisitResult.CONTINUE - } - override def postVisitDirectory(dir: Path, e: IOException): FileVisitResult = - Option(e) match { - case Some(ex) => throw ex - case None => - Files.deleteIfExists(dir) - FileVisitResult.CONTINUE + Files.walkFileTree( + dir, + new SimpleFileVisitor[Path]() { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + Files.deleteIfExists(file) + count.incrementAndGet() + FileVisitResult.CONTINUE } - }) + override def postVisitDirectory(dir: Path, e: IOException): FileVisitResult = + Option(e) match { + case Some(ex) => throw ex + case None => + Files.deleteIfExists(dir) + FileVisitResult.CONTINUE + } + } + ) count.get } @@ -44,12 +47,14 @@ object File { if (Files.isDirectory(path)) deleteDirectory(path) else deleteFile(path).map(_ => 1) - def withTempDir[F[_]: Sync, A](parent: Path, prefix: String) - (f: Path => Stream[F, A]): Stream[F, A] = + def withTempDir[F[_]: Sync, A](parent: Path, prefix: String)( + f: Path => Stream[F, A] + ): Stream[F, A] = Stream.bracket(mkTempDir(parent, prefix))(p => delete(p).map(_ => ())).flatMap(f) def listFiles[F[_]: Sync](pred: Path => Boolean, dir: Path): F[List[Path]] = Sync[F].delay { - val javaList = Files.list(dir).filter(p => pred(p)).collect(java.util.stream.Collectors.toList()) + val javaList = + Files.list(dir).filter(p => pred(p)).collect(java.util.stream.Collectors.toList()) javaList.asScala.toList.sortBy(_.getFileName.toString) } diff --git a/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala b/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala index 497b1f33..23c39f16 100644 --- a/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala +++ b/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala @@ -1,8 +1,6 @@ package docspell.text.ocr -case class MimeTypeHint(filename: Option[String], advertised: Option[String]) { - -} +case class MimeTypeHint(filename: Option[String], advertised: Option[String]) {} object MimeTypeHint { val none = MimeTypeHint(None, None) diff --git a/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala b/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala index b25de191..884a1581 100644 --- a/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala +++ b/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala @@ -6,12 +6,23 @@ import fs2.Stream object TextExtract { - def extract[F[_]: Sync: ContextShift](in: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = + def extract[F[_]: Sync: ContextShift]( + in: Stream[F, Byte], + blocker: Blocker, + lang: String, + config: Config + ): Stream[F, String] = extractOCR(in, blocker, lang, config) - def extractOCR[F[_]: Sync: ContextShift](in: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = - Stream.eval(TikaMimetype.detect(in)). - flatMap({ + def extractOCR[F[_]: Sync: ContextShift]( + in: Stream[F, Byte], + blocker: Blocker, + lang: String, + config: Config + ): Stream[F, String] = + Stream + .eval(TikaMimetype.detect(in)) + .flatMap({ case mt if !config.isAllowed(mt) => raiseError(s"File `$mt` not allowed") diff --git a/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala b/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala index cd8918bb..7a7cd292 100644 --- a/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala +++ b/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala @@ -11,20 +11,16 @@ object TextSplitter { def split[F[_]](str: String, sep: Set[Char], start: Int = 0): Stream[F, Word] = { val indexes = sep.map(c => str.indexOf(c.toInt)).filter(_ >= 0) - val index = if (indexes.isEmpty) - 1 else indexes.min + val index = if (indexes.isEmpty) -1 else indexes.min if (index < 0) Stream.emit(Word(str, start, start + str.length)) else if (index == 0) split(str.substring(1), sep, start + 1) - else Stream.emit(Word(str.substring(0, index), start, start + index)) ++ - Stream.suspend(split(str.substring(index + 1), sep, start + index + 1)) + else + Stream.emit(Word(str.substring(0, index), start, start + index)) ++ + Stream.suspend(split(str.substring(index + 1), sep, start + index + 1)) } - - def splitToken[F[_]](str: String, sep: Set[Char], start: Int = 0): Stream[F, Word] = { - split(str, sep, start). - map(w => w.trim(trimChars)). - filter(_.nonEmpty). - map(_.toLower) - } + def splitToken[F[_]](str: String, sep: Set[Char], start: Int = 0): Stream[F, Word] = + split(str, sep, start).map(w => w.trim(trimChars)).filter(_.nonEmpty).map(_.toLower) } diff --git a/modules/text/src/main/scala/docspell/text/split/Word.scala b/modules/text/src/main/scala/docspell/text/split/Word.scala index 15587f7d..88f0c3c0 100644 --- a/modules/text/src/main/scala/docspell/text/split/Word.scala +++ b/modules/text/src/main/scala/docspell/text/split/Word.scala @@ -1,9 +1,9 @@ package docspell.text.split case class Word(value: String, begin: Int, end: Int) { - def isEmpty: Boolean = value.isEmpty + def isEmpty: Boolean = value.isEmpty def nonEmpty: Boolean = !isEmpty - def length : Int = value.length + def length: Int = value.length def trimLeft(chars: Set[Char]): Word = { val v = value.dropWhile(chars.contains) diff --git a/modules/text/src/test/scala/docspell/text/TestFiles.scala b/modules/text/src/test/scala/docspell/text/TestFiles.scala index 90b426f4..a33bf4b9 100644 --- a/modules/text/src/test/scala/docspell/text/TestFiles.scala +++ b/modules/text/src/test/scala/docspell/text/TestFiles.scala @@ -7,20 +7,21 @@ import fs2.Stream import scala.concurrent.ExecutionContext object TestFiles { - val blocker = Blocker.liftExecutionContext(ExecutionContext.global) + val blocker = Blocker.liftExecutionContext(ExecutionContext.global) implicit val CS = IO.contextShift(ExecutionContext.global) - val letterSourceDE: Stream[IO, Byte] = - LenientUri.fromJava(getClass.getResource("/letter-de-source.pdf")). - readURL[IO](16 * 1024, blocker) + LenientUri + .fromJava(getClass.getResource("/letter-de-source.pdf")) + .readURL[IO](16 * 1024, blocker) val letterSourceEN: Stream[IO, Byte] = - LenientUri.fromJava(getClass.getResource("/letter-en-source.pdf")). - readURL[IO](16 * 1024, blocker) + LenientUri + .fromJava(getClass.getResource("/letter-en-source.pdf")) + .readURL[IO](16 * 1024, blocker) - - val letterDEText = """Max Mustermann + val letterDEText = + """Max Mustermann | |Lilienweg 21 | @@ -52,7 +53,8 @@ object TestFiles { |Max Mustermann |""".stripMargin.trim - val letterENText = """Derek Jeter + val letterENText = + """Derek Jeter | |123 Elm Ave. | diff --git a/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala b/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala index fbb1c5a1..721d2c35 100644 --- a/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala +++ b/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala @@ -22,11 +22,9 @@ object ContactAnnotateSpec extends SimpleTestSuite { val labels = Contact.annotate(text) assertEquals(labels.size, 2) - assertEquals(labels(0), - NerLabel("john.smith@example.com", NerTag.Email, 25, 47)) + assertEquals(labels(0), NerLabel("john.smith@example.com", NerTag.Email, 25, 47)) assertEquals(text.substring(25, 47).toLowerCase, "john.smith@example.com") - assertEquals(labels(1), - NerLabel("example.com", NerTag.Website, 308, 319)) + assertEquals(labels(1), NerLabel("example.com", NerTag.Website, 308, 319)) assertEquals(text.substring(308, 319).toLowerCase, "example.com") } } diff --git a/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala b/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala index b73471e7..309c241b 100644 --- a/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala +++ b/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala @@ -8,45 +8,49 @@ object TextAnalyserSuite extends SimpleTestSuite { test("find english ner labels") { val labels = StanfordNerClassifier.nerAnnotate(Language.English)(TestFiles.letterENText) - val expect = Vector(NerLabel("Derek",NerTag.Person,0,5) - , NerLabel("Jeter",NerTag.Person,6,11) - , NerLabel("Treesville",NerTag.Person,27,37) - , NerLabel("Derek",NerTag.Person,69,74) - , NerLabel("Jeter",NerTag.Person,75,80) - , NerLabel("Treesville",NerTag.Location,96,106) - , NerLabel("M.",NerTag.Person,142,144) - , NerLabel("Leat",NerTag.Person,145,149) - , NerLabel("Syrup",NerTag.Organization,160,165) - , NerLabel("Production",NerTag.Organization,166,176) - , NerLabel("Old",NerTag.Organization,177,180) - , NerLabel("Sticky",NerTag.Organization,181,187) - , NerLabel("Pancake",NerTag.Organization,188,195) - , NerLabel("Company",NerTag.Organization,196,203) - , NerLabel("Maple",NerTag.Location,208,213) - , NerLabel("Lane",NerTag.Location,214,218) - , NerLabel("Forest",NerTag.Location,220,226) - , NerLabel("Hemptown",NerTag.Location,241,249) - , NerLabel("Little",NerTag.Organization,349,355) - , NerLabel("League",NerTag.Organization,356,362) - , NerLabel("Derek",NerTag.Person,1119,1124) - , NerLabel("Jeter",NerTag.Person,1125,1130)) + val expect = Vector( + NerLabel("Derek", NerTag.Person, 0, 5), + NerLabel("Jeter", NerTag.Person, 6, 11), + NerLabel("Treesville", NerTag.Person, 27, 37), + NerLabel("Derek", NerTag.Person, 69, 74), + NerLabel("Jeter", NerTag.Person, 75, 80), + NerLabel("Treesville", NerTag.Location, 96, 106), + NerLabel("M.", NerTag.Person, 142, 144), + NerLabel("Leat", NerTag.Person, 145, 149), + NerLabel("Syrup", NerTag.Organization, 160, 165), + NerLabel("Production", NerTag.Organization, 166, 176), + NerLabel("Old", NerTag.Organization, 177, 180), + NerLabel("Sticky", NerTag.Organization, 181, 187), + NerLabel("Pancake", NerTag.Organization, 188, 195), + NerLabel("Company", NerTag.Organization, 196, 203), + NerLabel("Maple", NerTag.Location, 208, 213), + NerLabel("Lane", NerTag.Location, 214, 218), + NerLabel("Forest", NerTag.Location, 220, 226), + NerLabel("Hemptown", NerTag.Location, 241, 249), + NerLabel("Little", NerTag.Organization, 349, 355), + NerLabel("League", NerTag.Organization, 356, 362), + NerLabel("Derek", NerTag.Person, 1119, 1124), + NerLabel("Jeter", NerTag.Person, 1125, 1130) + ) assertEquals(labels, expect) } test("find german ner labels") { val labels = StanfordNerClassifier.nerAnnotate(Language.German)(TestFiles.letterDEText) - val expect = Vector(NerLabel("Max", NerTag.Person, 0, 3) - , NerLabel("Mustermann", NerTag.Person, 4, 14) - , NerLabel("Lilienweg", NerTag.Location, 16, 25) - , NerLabel("Max", NerTag.Person, 77, 80) - , NerLabel("Mustermann", NerTag.Person, 81, 91) - , NerLabel("Lilienweg", NerTag.Location, 93, 102) - , NerLabel("EasyCare", NerTag.Organization, 124, 132) - , NerLabel("AG", NerTag.Organization, 133, 135) - , NerLabel("Ackerweg", NerTag.Location, 158, 166) - , NerLabel("Nebendorf", NerTag.Location, 184, 193) - , NerLabel("Max", NerTag.Person, 505, 508) - , NerLabel("Mustermann", NerTag.Person, 509, 519)) + val expect = Vector( + NerLabel("Max", NerTag.Person, 0, 3), + NerLabel("Mustermann", NerTag.Person, 4, 14), + NerLabel("Lilienweg", NerTag.Location, 16, 25), + NerLabel("Max", NerTag.Person, 77, 80), + NerLabel("Mustermann", NerTag.Person, 81, 91), + NerLabel("Lilienweg", NerTag.Location, 93, 102), + NerLabel("EasyCare", NerTag.Organization, 124, 132), + NerLabel("AG", NerTag.Organization, 133, 135), + NerLabel("Ackerweg", NerTag.Location, 158, 166), + NerLabel("Nebendorf", NerTag.Location, 184, 193), + NerLabel("Max", NerTag.Person, 505, 508), + NerLabel("Mustermann", NerTag.Person, 509, 519) + ) assertEquals(labels, expect) } } diff --git a/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala b/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala index 40b46a77..f9e94ec2 100644 --- a/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala +++ b/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala @@ -9,16 +9,22 @@ object TextExtractionSuite extends SimpleTestSuite { test("extract english pdf") { ignore() - val text = TextExtract.extract[IO](letterSourceEN, blocker, "eng", Config.default). - compile.lastOrError.unsafeRunSync() + val text = TextExtract + .extract[IO](letterSourceEN, blocker, "eng", Config.default) + .compile + .lastOrError + .unsafeRunSync() println(text) } test("extract german pdf") { ignore() val expect = TestFiles.letterDEText - val extract = TextExtract.extract[IO](letterSourceDE, blocker, "deu", Config.default). - compile.lastOrError.unsafeRunSync() + val extract = TextExtract + .extract[IO](letterSourceDE, blocker, "deu", Config.default) + .compile + .lastOrError + .unsafeRunSync() assertEquals(extract.trim, expect.trim) } diff --git a/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala b/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala index 004d329c..13e91a5d 100644 --- a/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala +++ b/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala @@ -15,7 +15,6 @@ object TestSplitterSpec extends SimpleTestSuite { val words = TextSplitter.splitToken(text, " \t\r\n".toSet).toVector - assertEquals(words.size, 31) assertEquals(words(13), Word("bitte", 109, 114)) assertEquals(text.substring(109, 114).toLowerCase, "bitte")