Sketch a scheduler for running periodic tasks

Periodic tasks are special in that they are usually kept around and
started based on a schedule. A new component checks periodic tasks and
submits them in the queue once they are due.

In order to avoid duplicate periodic jobs, the tracker of a job is
used to store the periodic job id. Each time a periodic task is due,
it is first checked if there is a job running (or queued) for this
task.

modules/store/src/main/resources/db/migration/mariadb/V1.3.0__periodic_job.sql

@@ -0,0 +1,15 @@
+CREATE TABLE `periodic_task` (
+  `id` varchar(254) not null primary key,
+  `enabled` boolean not null,
+  `task` varchar(254) not null,
+  `group_` varchar(254) not null,
+  `args` text not null,
+  `subject` varchar(254) not null,
+  `submitter` varchar(254) not null,
+  `priority` int not null,
+  `worker` varchar(254),
+  `marked` timestamp,
+  `timer` varchar(254) not null,
+  `nextrun` timestamp not null,
+  `created` timestamp not null
+);

modules/store/src/main/resources/db/migration/postgresql/V1.3.0__periodic_job.sql

@@ -0,0 +1,13 @@
+CREATE TABLE "periodic_task" (
+  "id" varchar(254) not null primary key,
+  "enabled" boolean not null,
+  "task" varchar(254) not null,
+  "group_" varchar(254) not null,
+  "args" text not null,
+  "subject" varchar(254) not null,
+  "submitter" varchar(254) not null,
+  "priority" int not null,
+  "worker" varchar(254),
+  "timer" varchar(254) not null,
+  "nextrun" timestamp not null
+);

modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala

@@ -7,6 +7,7 @@ import doobie._
 import doobie.implicits.legacy.instant._
 import doobie.util.log.Success
 import emil.{MailAddress, SSLType}
+import com.github.eikek.calev.CalEvent
 
 import docspell.common._
 import docspell.common.syntax.all._
@@ -98,6 +99,9 @@ trait DoobieMeta {
     Meta[String].imap(str => str.split(',').toList.map(_.trim).map(EmilUtil.unsafeReadMailAddress))(
       lma => lma.map(EmilUtil.mailAddressString).mkString(",")
     )
+
+  implicit val metaCalEvent: Meta[CalEvent] =
+    Meta[String].timap(CalEvent.unsafe)(_.asString)
 }
 
 object DoobieMeta extends DoobieMeta {

modules/store/src/main/scala/docspell/store/queue/PeriodicTaskStore.scala

@@ -0,0 +1,77 @@
+package docspell.store.queue
+
+import cats.effect._
+import cats.implicits._
+import fs2.Stream
+import docspell.common._
+import docspell.store.Store
+import docspell.store.records._
+
+trait PeriodicTaskStore[F[_]] {
+
+  /** Get the free periodic task due next and reserve it to the given
+    * worker.
+    *
+    * If found, the task is returned and resource finalization takes
+    * care of unmarking the task after use and updating `nextRun` with
+    * the next timestamp.
+    */
+  def takeNext(worker: Ident): Resource[F, Option[RPeriodicTask]]
+
+  def clearMarks(name: Ident): F[Unit]
+
+  def findNonFinalJob(pjobId: Ident): F[Option[RJob]]
+
+  def insert(task: RPeriodicTask): F[Unit]
+}
+
+object PeriodicTaskStore {
+
+  def create[F[_]: Sync](store: Store[F]): Resource[F, PeriodicTaskStore[F]] =
+    Resource.pure[F, PeriodicTaskStore[F]](new PeriodicTaskStore[F] {
+      println(s"$store")
+
+      def takeNext(worker: Ident): Resource[F, Option[RPeriodicTask]] = {
+        val chooseNext: F[Either[String, Option[RPeriodicTask]]] =
+          getNext.flatMap {
+            case Some(pj) =>
+              mark(pj.id, worker).map {
+                case true  => Right(Some(pj.copy(worker = worker.some)))
+                case false => Left("Cannot mark periodic task")
+              }
+            case None =>
+              val result: Either[String, Option[RPeriodicTask]] =
+                Right(None)
+              result.pure[F]
+          }
+        val get =
+          Stream.eval(chooseNext).repeat.take(10).find(_.isRight).compile.lastOrError
+        val r = Resource.make(get)({
+          case Right(Some(pj)) => unmark(pj)
+          case _               => ().pure[F]
+        })
+        r.flatMap {
+          case Right(job) => Resource.pure(job)
+          case Left(err)  => Resource.liftF(Sync[F].raiseError(new Exception(err)))
+        }
+      }
+
+      def getNext: F[Option[RPeriodicTask]] =
+        Sync[F].raiseError(new Exception("not implemented"))
+
+      def mark(pid: Ident, name: Ident): F[Boolean] =
+        Sync[F].raiseError(new Exception(s"not implemented $pid $name"))
+
+      def unmark(job: RPeriodicTask): F[Unit] =
+        Sync[F].raiseError(new Exception(s"not implemented $job"))
+
+      def clearMarks(name: Ident): F[Unit] =
+        Sync[F].raiseError(new Exception("not implemented"))
+
+      def findNonFinalJob(pjobId: Ident): F[Option[RJob]] =
+        Sync[F].raiseError(new Exception("not implemented"))
+
+      def insert(task: RPeriodicTask): F[Unit] =
+        Sync[F].raiseError(new Exception("not implemented"))
+    })
+}

modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala

@@ -0,0 +1,141 @@
+package docspell.store.records
+
+import cats.effect._
+import cats.implicits._
+import doobie._
+import doobie.implicits._
+import com.github.eikek.calev.CalEvent
+import docspell.common._
+import docspell.store.impl.Column
+import docspell.store.impl.Implicits._
+
+/** A periodic task is a special job description, that shares a few
+  * properties of a `RJob`. It must provide all information to create
+  * a `RJob` value eventually.
+  */
+case class RPeriodicTask(
+    id: Ident,
+    enabled: Boolean,
+    task: Ident,
+    group: Ident,
+    args: String,
+    subject: String,
+    submitter: Ident,
+    priority: Priority,
+    worker: Option[Ident],
+    marked: Option[Timestamp],
+    timer: CalEvent,
+    nextrun: Timestamp,
+    created: Timestamp
+) {
+
+  def toJob[F[_]: Sync]: F[RJob] =
+    for {
+      now <- Timestamp.current[F]
+      jid <- Ident.randomId[F]
+    } yield RJob(
+      jid,
+      task,
+      group,
+      args,
+      subject,
+      now,
+      submitter,
+      priority,
+      JobState.Waiting,
+      0,
+      0,
+      Some(id),
+      None,
+      None,
+      None
+    )
+}
+
+object RPeriodicTask {
+
+  def create[F[_]: Sync](
+      enabled: Boolean,
+      task: Ident,
+      group: Ident,
+      args: String,
+      subject: String,
+      submitter: Ident,
+      priority: Priority,
+      worker: Option[Ident],
+      marked: Option[Timestamp],
+      timer: CalEvent
+  ): F[RPeriodicTask] =
+    Ident
+      .randomId[F]
+      .flatMap(id =>
+        Timestamp
+          .current[F]
+          .map { now =>
+            RPeriodicTask(
+              id,
+              enabled,
+              task,
+              group,
+              args,
+              subject,
+              submitter,
+              priority,
+              worker,
+              marked,
+              timer,
+              timer
+                .nextElapse(now.atZone(Timestamp.UTC))
+                .map(_.toInstant)
+                .map(Timestamp.apply)
+                .getOrElse(Timestamp.Epoch),
+              now
+            )
+          }
+      )
+
+  val table = fr"periodic_task"
+
+  object Columns {
+    val id        = Column("id")
+    val enabled   = Column("enabled")
+    val task      = Column("task")
+    val group     = Column("group_")
+    val args      = Column("args")
+    val subject   = Column("subject")
+    val submitter = Column("submitter")
+    val priority  = Column("priority")
+    val worker    = Column("worker")
+    val marked    = Column("marked")
+    val timer     = Column("timer")
+    val nextrun   = Column("nextrun")
+    val created   = Column("created")
+    val all = List(
+      id,
+      enabled,
+      task,
+      group,
+      args,
+      subject,
+      submitter,
+      priority,
+      worker,
+      marked,
+      timer,
+      nextrun,
+      created
+    )
+  }
+
+  import Columns._
+
+  def insert(v: RPeriodicTask): ConnectionIO[Int] = {
+    val sql = insertRow(
+      table,
+      all,
+      fr"${v.id},${v.enabled},${v.task},${v.group},${v.args},${v.subject},${v.submitter},${v.priority},${v.worker},${v.marked},${v.timer},${v.nextrun},${v.created}"
+    )
+    sql.update.run
+  }
+
+}