TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-21 18:08:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate periodic tasks

Browse Source 
The first use case for periodic task is the cleanup of expired
invitation keys. This is part of a house-keeping periodic task.
This commit is contained in:
Eike Kettner
2020-03-08 15:26:56 +01:00
parent 616c333fa5
commit 854a596da3
25 changed files with 388 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modules/store/src/main/scala/docspell/store/impl/Column.scala
View File

@ -30,6 +30,9 @@ case class Column(name: String, ns: String = "", alias: String = "") {
def is(c: Column): Fragment =
f ++ fr"=" ++ c.f
def isNot[A: Put](value: A): Fragment =
f ++ fr"<> $value"
def isNull: Fragment =
f ++ fr"is null"

26
modules/store/src/main/scala/docspell/store/queries/QPeriodicTask.scala
View File

@ -1,26 +1,27 @@
package docspell.store.queries
//import cats.implicits._
import docspell.common._
//import docspell.common.syntax.all._
import docspell.store.impl.Implicits._
import docspell.store.records._
import doobie._
import doobie.implicits._
//import org.log4s._
object QPeriodicTask {
// private[this] val logger = getLogger
def clearWorkers(name: Ident): ConnectionIO[Int] = {
val worker = RPeriodicTask.Columns.worker
updateRow(RPeriodicTask.table, worker.is(name), worker.setTo[Ident](None)).update.run
}
def setWorker(pid: Ident, name: Ident): ConnectionIO[Int] = {
def setWorker(pid: Ident, name: Ident, ts: Timestamp): ConnectionIO[Int] = {
val id = RPeriodicTask.Columns.id
val worker = RPeriodicTask.Columns.worker
updateRow(RPeriodicTask.table, and(id.is(pid), worker.isNull), worker.setTo(name)).update.run
val marked = RPeriodicTask.Columns.marked
updateRow(
RPeriodicTask.table,
and(id.is(pid), worker.isNull),
commas(worker.setTo(name), marked.setTo(ts))
).update.run
}
def unsetWorker(
@ -37,10 +38,17 @@ object QPeriodicTask {
).update.run
}
def findNext: ConnectionIO[Option[RPeriodicTask]] = {
val order = orderBy(RPeriodicTask.Columns.nextrun.f) ++ fr"ASC"
def findNext(excl: Option[Ident]): ConnectionIO[Option[RPeriodicTask]] = {
val enabled = RPeriodicTask.Columns.enabled
val pid = RPeriodicTask.Columns.id
val order = orderBy(RPeriodicTask.Columns.nextrun.f) ++ fr"ASC"
val where = excl match {
case Some(id) => and(pid.isNot(id), enabled.is(true))
case None => enabled.is(true)
}
val sql =
selectSimple(RPeriodicTask.Columns.all, RPeriodicTask.table, Fragment.empty) ++ order
selectSimple(RPeriodicTask.Columns.all, RPeriodicTask.table, where) ++ order
sql.query[RPeriodicTask].streamWithChunkSize(2).take(1).compile.last
}

2
modules/store/src/main/scala/docspell/store/queue/JobQueue.scala
View File

@ -29,7 +29,7 @@ object JobQueue {
worker: Ident,
retryPause: Duration
): F[Option[RJob]] =
logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
logger.ftrace("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
def insert(job: RJob): F[Unit] =
store

17
modules/store/src/main/scala/docspell/store/queue/Marked.scala Normal file
View File

@ -0,0 +1,17 @@
package docspell.store.queue
sealed trait Marked[+A] {}
object Marked {
final case class Found[A](value: A) extends Marked[A]
final case object NotFound extends Marked[Nothing]
final case object NotMarkable extends Marked[Nothing]
def found[A](v: A): Marked[A] = Found(v)
def notFound[A]: Marked[A] = NotFound
def notMarkable[A]: Marked[A] = NotMarkable
}

66
modules/store/src/main/scala/docspell/store/queue/PeriodicTaskStore.scala
View File

@ -2,7 +2,6 @@ package docspell.store.queue
import cats.effect._
import cats.implicits._
import fs2.Stream
import org.log4s.getLogger
import com.github.eikek.fs2calev._
import docspell.common._
@ -20,7 +19,10 @@ trait PeriodicTaskStore[F[_]] {
* care of unmarking the task after use and updating `nextRun` with
* the next timestamp.
*/
def takeNext(worker: Ident): Resource[F, Option[RPeriodicTask]]
def takeNext(
worker: Ident,
excludeId: Option[Ident]
): Resource[F, Marked[RPeriodicTask]]
def clearMarks(name: Ident): F[Unit]
@ -33,6 +35,8 @@ trait PeriodicTaskStore[F[_]] {
/** Adds the task only if it not already exists.
*/
def add(task: RPeriodicTask): F[AddResult]
def findJoexNodes: F[Vector[RNode]]
}
object PeriodicTaskStore {
@ -40,38 +44,37 @@ object PeriodicTaskStore {
def create[F[_]: Sync](store: Store[F]): Resource[F, PeriodicTaskStore[F]] =
Resource.pure[F, PeriodicTaskStore[F]](new PeriodicTaskStore[F] {
println(s"$store")
def takeNext(worker: Ident): Resource[F, Option[RPeriodicTask]] = {
val chooseNext: F[Either[String, Option[RPeriodicTask]]] =
getNext.flatMap {
def takeNext(
worker: Ident,
excludeId: Option[Ident]
): Resource[F, Marked[RPeriodicTask]] = {
val chooseNext: F[Marked[RPeriodicTask]] =
getNext(excludeId).flatMap {
case Some(pj) =>
mark(pj.id, worker).map {
case true => Right(Some(pj.copy(worker = worker.some)))
case false => Left("Cannot mark periodic task")
case true => Marked.found(pj.copy(worker = worker.some))
case false => Marked.notMarkable
}
case None =>
val result: Either[String, Option[RPeriodicTask]] =
Right(None)
result.pure[F]
Marked.notFound.pure[F]
}
val get =
Stream.eval(chooseNext).repeat.take(10).find(_.isRight).compile.lastOrError
val r = Resource.make(get)({
case Right(Some(pj)) => unmark(pj)
case _ => ().pure[F]
Resource.make(chooseNext)({
case Marked.Found(pj) => unmark(pj)
case _ => ().pure[F]
})
r.flatMap {
case Right(job) => Resource.pure(job)
case Left(err) => Resource.liftF(Sync[F].raiseError(new Exception(err)))
}
}
def getNext: F[Option[RPeriodicTask]] =
store.transact(QPeriodicTask.findNext)
def getNext(excl: Option[Ident]): F[Option[RPeriodicTask]] =
store.transact(QPeriodicTask.findNext(excl))
def mark(pid: Ident, name: Ident): F[Boolean] =
store.transact(QPeriodicTask.setWorker(pid, name)).map(_ > 0)
Timestamp
.current[F]
.flatMap(now =>
store.transact(QPeriodicTask.setWorker(pid, name, now)).map(_ > 0)
)
def unmark(job: RPeriodicTask): F[Unit] =
for {
@ -98,16 +101,15 @@ object PeriodicTaskStore {
def insert(task: RPeriodicTask): F[Unit] = {
val update = store.transact(RPeriodicTask.update(task))
val insertAttempt = store.transact(RPeriodicTask.insert(task))
.attempt.map {
case Right(n) => n > 0
case Left(_) => false
}
val insertAttempt = store.transact(RPeriodicTask.insert(task)).attempt.map {
case Right(n) => n > 0
case Left(_) => false
}
for {
n1 <- update
n1 <- update
ins <- if (n1 == 0) insertAttempt else true.pure[F]
_ <- if (ins) 1.pure[F] else update
_ <- if (ins) 1.pure[F] else update
} yield ()
}
@ -116,5 +118,9 @@ object PeriodicTaskStore {
val exists = RPeriodicTask.exists(task.id)
store.add(insert, exists)
}
def findJoexNodes: F[Vector[RNode]] =
store.transact(RNode.findAll(NodeType.Joex))
})
}

3
modules/store/src/main/scala/docspell/store/records/RInvitation.scala
View File

@ -46,4 +46,7 @@ object RInvitation {
_ <- delete(invite)
} yield inv > 0
}
def deleteOlderThan(ts: Timestamp): ConnectionIO[Int] =
deleteFrom(table, created.isLt(ts)).update.run
}

19
modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala
View File

@ -8,6 +8,7 @@ import com.github.eikek.calev.CalEvent
import docspell.common._
import docspell.store.impl.Column
import docspell.store.impl.Implicits._
import io.circe.Encoder
/** A periodic task is a special job description, that shares a few
* properties of a `RJob`. It must provide all information to create
@ -62,8 +63,6 @@ object RPeriodicTask {
subject: String,
submitter: Ident,
priority: Priority,
worker: Option[Ident],
marked: Option[Timestamp],
timer: CalEvent
): F[RPeriodicTask] =
Ident
@ -81,8 +80,8 @@ object RPeriodicTask {
subject,
submitter,
priority,
worker,
marked,
None,
None,
timer,
timer
.nextElapse(now.atZone(Timestamp.UTC))
@ -94,6 +93,18 @@ object RPeriodicTask {
}
)
def createJson[F[_]: Sync, A](
enabled: Boolean,
task: Ident,
group: Ident,
args: A,
subject: String,
submitter: Ident,
priority: Priority,
timer: CalEvent
)(implicit E: Encoder[A]): F[RPeriodicTask] =
create[F](enabled, task, group, E(args).noSpaces, subject, submitter, priority, timer)
val table = fr"periodic_task"
object Columns {