Move job queue to scheduler-api and fix notification of periodic tasks

2025-06-22 10:28:27 +00:00 · 2022-03-12 15:31:27 +01:00
parent aafd908906
commit 83d3644b39
31 changed files with 108 additions and 103 deletions
--- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala
+++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala
@ -7,19 +7,17 @@
 package docspell.backend

 import cats.effect._
-
 import docspell.backend.auth.Login
 import docspell.backend.fulltext.CreateIndex
-import docspell.backend.msg.JobQueuePublish
 import docspell.backend.ops._
 import docspell.backend.signup.OSignup
 import docspell.ftsclient.FtsClient
 import docspell.notification.api.{EventExchange, NotificationModule}
 import docspell.pubsub.api.PubSubT
+import docspell.scheduler.msg.JobQueuePublish
 import docspell.store.Store
 import docspell.store.usertask.UserTaskStore
 import docspell.totp.Totp
-
 import emil.Emil

 trait BackendApp[F[_]] {
--- a/modules/backend/src/main/scala/docspell/backend/msg/CancelJob.scala
+++ b/modules/backend/src/main/scala/docspell/backend/msg/CancelJob.scala
@ -1,27 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.backend.msg
-
-import docspell.common._
-import docspell.pubsub.api.{Topic, TypedTopic}
-
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import io.circe.{Decoder, Encoder}
-
-/** Message to request to cancel a job. */
-final case class CancelJob(jobId: Ident, nodeId: Ident)
-
-object CancelJob {
-  implicit val jsonDecoder: Decoder[CancelJob] =
-    deriveDecoder[CancelJob]
-
-  implicit val jsonEncoder: Encoder[CancelJob] =
-    deriveEncoder[CancelJob]
-
-  val topic: TypedTopic[CancelJob] =
-    TypedTopic(Topic("job-cancel-request"))
-}
--- a/modules/backend/src/main/scala/docspell/backend/msg/JobQueuePublish.scala
+++ b/modules/backend/src/main/scala/docspell/backend/msg/JobQueuePublish.scala
@ -1,80 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.backend.msg
-
-import cats.effect._
-import cats.implicits._
-
-import docspell.common.{Duration, Ident, Priority}
-import docspell.notification.api.Event
-import docspell.notification.api.EventSink
-import docspell.pubsub.api.PubSubT
-import docspell.store.Store
-import docspell.store.queue.JobQueue
-import docspell.store.records.RJob
-
-final class JobQueuePublish[F[_]: Sync](
-    delegate: JobQueue[F],
-    pubsub: PubSubT[F],
-    eventSink: EventSink[F]
-) extends JobQueue[F] {
-
-  private def msg(job: RJob): JobSubmitted =
-    JobSubmitted(job.id, job.group, job.task, job.args)
-
-  private def event(job: RJob): Event.JobSubmitted =
-    Event.JobSubmitted(
-      job.id,
-      job.group,
-      job.task,
-      job.args,
-      job.state,
-      job.subject,
-      job.submitter
-    )
-
-  private def publish(job: RJob): F[Unit] =
-    pubsub.publish1(JobSubmitted.topic, msg(job)).as(()) *>
-      eventSink.offer(event(job))
-
-  def insert(job: RJob) =
-    delegate.insert(job).flatTap(_ => publish(job))
-
-  def insertIfNew(job: RJob) =
-    delegate.insertIfNew(job).flatTap {
-      case true  => publish(job)
-      case false => ().pure[F]
-    }
-
-  def insertAll(jobs: Seq[RJob]) =
-    delegate.insertAll(jobs).flatTap { results =>
-      results.zip(jobs).traverse { case (res, job) =>
-        if (res) publish(job)
-        else ().pure[F]
-      }
-    }
-
-  def insertAllIfNew(jobs: Seq[RJob]) =
-    delegate.insertAllIfNew(jobs).flatTap { results =>
-      results.zip(jobs).traverse { case (res, job) =>
-        if (res) publish(job)
-        else ().pure[F]
-      }
-    }
-
-  def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration) =
-    delegate.nextJob(prio, worker, retryPause)
-}
-
-object JobQueuePublish {
-  def apply[F[_]: Async](
-      store: Store[F],
-      pubSub: PubSubT[F],
-      eventSink: EventSink[F]
-  ): Resource[F, JobQueue[F]] =
-    JobQueue(store).map(q => new JobQueuePublish[F](q, pubSub, eventSink))
-}
--- a/modules/backend/src/main/scala/docspell/backend/msg/JobSubmitted.scala
+++ b/modules/backend/src/main/scala/docspell/backend/msg/JobSubmitted.scala
@ -1,26 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.backend.msg
-import docspell.common._
-import docspell.pubsub.api.{Topic, TypedTopic}
-
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import io.circe.{Decoder, Encoder}
-
-final case class JobSubmitted(jobId: Ident, group: Ident, task: Ident, args: String)
-
-object JobSubmitted {
-
-  implicit val jsonDecoder: Decoder[JobSubmitted] =
-    deriveDecoder
-
-  implicit val jsonEncoder: Encoder[JobSubmitted] =
-    deriveEncoder
-
-  val topic: TypedTopic[JobSubmitted] =
-    TypedTopic(Topic("job-submitted"))
-}
--- a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala
+++ b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala
@ -7,17 +7,19 @@
 package docspell.backend.msg

 import cats.data.NonEmptyList
-import docspell.pubsub.api.{Topic, TypedTopic}
-import docspell.scheduler.msg.JobDone
+import docspell.pubsub.api.TypedTopic
+import docspell.scheduler.msg._

 /** All topics used in Docspell. */
 object Topics {

-  /** A generic notification to the job executors to look for new work. */
-  val jobsNotify: TypedTopic[Unit] =
-    TypedTopic[Unit](Topic("jobs-notify"))
-
  /** A list of all topics. It is required to list every topic in use here! */
  val all: NonEmptyList[TypedTopic[_]] =
-    NonEmptyList.of(JobDone.topic, CancelJob.topic, jobsNotify, JobSubmitted.topic)
+    NonEmptyList.of(
+      JobDone.topic,
+      CancelJob.topic,
+      JobsNotify(),
+      JobSubmitted.topic,
+      PeriodicTaskNotify()
+    )
 }
--- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala
@ -9,19 +9,17 @@ package docspell.backend.ops
 import cats.effect.{Async, Resource}
 import cats.implicits._
 import fs2.Stream
-
 import docspell.backend.JobFactory
 import docspell.backend.PasswordCrypt
 import docspell.backend.ops.OCollective._
 import docspell.common._
 import docspell.store.UpdateResult
 import docspell.store.queries.{QCollective, QUser}
-import docspell.store.queue.JobQueue
 import docspell.store.records._
 import docspell.store.usertask.{UserTask, UserTaskScope, UserTaskStore}
 import docspell.store.{AddResult, Store}
-
 import com.github.eikek.calev._
+import docspell.scheduler.JobQueue

 trait OCollective[F[_]] {

--- a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala
@ -9,14 +9,12 @@ package docspell.backend.ops
 import cats.data.OptionT
 import cats.effect._
 import cats.implicits._
-
 import docspell.backend.JobFactory
 import docspell.backend.ops.OFileRepository.IntegrityResult
 import docspell.common._
+import docspell.scheduler.JobQueue
 import docspell.store.Store
-import docspell.store.queue.JobQueue
 import docspell.store.records.RJob
-
 import scodec.bits.ByteVector

 trait OFileRepository[F[_]] {
--- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala
@ -10,15 +10,14 @@ import cats.data.NonEmptyList
 import cats.effect._
 import cats.implicits._
 import fs2.Stream
-
 import docspell.backend.JobFactory
 import docspell.backend.ops.OItemSearch._
 import docspell.common._
 import docspell.ftsclient._
 import docspell.query.ItemQuery._
 import docspell.query.ItemQueryDsl._
+import docspell.scheduler.JobQueue
 import docspell.store.queries.{QFolder, QItem, SelectedItem}
-import docspell.store.queue.JobQueue
 import docspell.store.records.RJob
 import docspell.store.{Store, qb}

--- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
@ -9,7 +9,6 @@ package docspell.backend.ops
 import cats.data.{NonEmptyList => Nel, OptionT}
 import cats.effect.{Async, Resource}
 import cats.implicits._
-
 import docspell.backend.AttachedEvent
 import docspell.backend.JobFactory
 import docspell.backend.fulltext.CreateIndex
@ -18,11 +17,10 @@ import docspell.common._
 import docspell.ftsclient.FtsClient
 import docspell.logging.Logger
 import docspell.notification.api.Event
+import docspell.scheduler.JobQueue
 import docspell.store.queries.{QAttachment, QItem, QMoveAttachment}
-import docspell.store.queue.JobQueue
 import docspell.store.records._
 import docspell.store.{AddResult, Store, UpdateResult}
-
 import doobie.implicits._

 trait OItem[F[_]] {
--- a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala
@ -9,11 +9,10 @@ package docspell.backend.ops
 import cats.data.OptionT
 import cats.effect._
 import cats.implicits._
-
-import docspell.backend.msg.JobDone
 import docspell.backend.ops.OJob.{CollectiveQueueState, JobCancelResult}
 import docspell.common._
 import docspell.pubsub.api.PubSubT
+import docspell.scheduler.msg.JobDone
 import docspell.store.Store
 import docspell.store.UpdateResult
 import docspell.store.queries.QJob
--- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala
@ -9,15 +9,16 @@ package docspell.backend.ops
 import cats.Applicative
 import cats.effect._
 import cats.implicits._
-
-import docspell.backend.msg.{CancelJob, Topics}
 import docspell.common.Ident
 import docspell.pubsub.api.PubSubT
+import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify}

 trait OJoex[F[_]] {

  def notifyAllNodes: F[Unit]

+  def notifyPeriodicTasks: F[Unit]
+
  def cancelJob(job: Ident, worker: Ident): F[Unit]
 }

@ -26,7 +27,10 @@ object OJoex {
    Resource.pure[F, OJoex[F]](new OJoex[F] {

      def notifyAllNodes: F[Unit] =
-        pubSub.publish1IgnoreErrors(Topics.jobsNotify, ()).as(())
+        pubSub.publish1IgnoreErrors(JobsNotify(), ()).void
+
+      def notifyPeriodicTasks: F[Unit] =
+        pubSub.publish1IgnoreErrors(PeriodicTaskNotify(), ()).void

      def cancelJob(job: Ident, worker: Ident): F[Unit] =
        pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(())
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@ -11,11 +11,10 @@ import cats.data.{EitherT, OptionT}
 import cats.effect._
 import cats.implicits._
 import fs2.Stream
-
 import docspell.backend.JobFactory
 import docspell.common._
+import docspell.scheduler.JobQueue
 import docspell.store.Store
-import docspell.store.queue.JobQueue
 import docspell.store.records._

 trait OUpload[F[_]] {
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala
@ -10,14 +10,12 @@ import cats.data.{NonEmptyList, OptionT}
 import cats.effect._
 import cats.implicits._
 import fs2.Stream
-
 import docspell.common._
 import docspell.notification.api.{ChannelRef, PeriodicDueItemsArgs, PeriodicQueryArgs}
+import docspell.scheduler.JobQueue
 import docspell.store.Store
-import docspell.store.queue.JobQueue
 import docspell.store.records.RNotificationChannel
 import docspell.store.usertask._
-
 import io.circe.Encoder

 trait OUserTask[F[_]] {
@ -98,7 +96,7 @@ object OUserTask {
          ptask <- task.encode.toPeriodicTask(scope, subject)
          job <- ptask.toJob
          _ <- queue.insert(job)
-          _ <- joex.notifyAllNodes
+          _ <- joex.notifyPeriodicTasks
        } yield ()

      def getScanMailbox(scope: UserTaskScope): Stream[F, UserTask[ScanMailboxArgs]] =
@ -124,7 +122,7 @@ object OUserTask {
      ): F[Unit] =
        for {
          _ <- taskStore.updateTask[ScanMailboxArgs](scope, subject, task)
-          _ <- joex.notifyAllNodes
+          _ <- joex.notifyPeriodicTasks
        } yield ()

      def getNotifyDueItems(
@ -153,7 +151,7 @@ object OUserTask {
      ): F[Unit] =
        for {
          _ <- taskStore.updateTask[PeriodicDueItemsArgs](scope, subject, task)
-          _ <- joex.notifyAllNodes
+          _ <- joex.notifyPeriodicTasks
        } yield ()

      def getPeriodicQuery(scope: UserTaskScope): Stream[F, UserTask[PeriodicQueryArgs]] =
@ -180,7 +178,7 @@ object OUserTask {
      ): F[Unit] =
        for {
          _ <- taskStore.updateTask[PeriodicQueryArgs](scope, subject, task)
-          _ <- joex.notifyAllNodes
+          _ <- joex.notifyPeriodicTasks
        } yield ()

      // When retrieving arguments containing channel references, we must update