Hooking the new pubsub impl into the application

modules/pubsub/api/src/main/scala/docspell/pubsub/api/PubSub.scala

@@ -6,9 +6,12 @@
 
 package docspell.pubsub.api
 
+import cats.Applicative
 import cats.data.NonEmptyList
 import fs2.{Pipe, Stream}
 
+import docspell.common.{Ident, Timestamp}
+
 import io.circe.Json
 
 trait PubSub[F[_]] {
@@ -18,3 +21,16 @@ trait PubSub[F[_]] {
 
   def subscribe(topics: NonEmptyList[Topic]): Stream[F, Message[Json]]
 }
+object PubSub {
+  def noop[F[_]: Applicative]: PubSub[F] =
+    new PubSub[F] {
+      def publish1(topic: Topic, msg: Json): F[MessageHead] =
+        Applicative[F].pure(MessageHead(Ident.unsafe("0"), Timestamp.Epoch, topic))
+
+      def publish(topic: Topic): Pipe[F, Json, MessageHead] =
+        _ => Stream.empty
+
+      def subscribe(topics: NonEmptyList[Topic]): Stream[F, Message[Json]] =
+        Stream.empty
+    }
+}

modules/pubsub/api/src/main/scala/docspell/pubsub/api/PubSubT.scala

@@ -7,6 +7,9 @@
 package docspell.pubsub.api
 
 import cats.data.NonEmptyList
+import cats.effect._
+import cats.implicits._
+import fs2.concurrent.SignallingRef
 import fs2.{Pipe, Stream}
 
 import docspell.common.Logger
@@ -15,22 +18,35 @@ trait PubSubT[F[_]] {
 
   def publish1[A](topic: TypedTopic[A], msg: A): F[MessageHead]
 
+  def publish1IgnoreErrors[A](topic: TypedTopic[A], msg: A): F[Unit]
+
   def publish[A](topic: TypedTopic[A]): Pipe[F, A, MessageHead]
 
   def subscribe[A](topic: TypedTopic[A]): Stream[F, Message[A]]
 
+  def subscribeSink[A](topic: TypedTopic[A])(handler: Message[A] => F[Unit]): F[F[Unit]]
+
   def delegate: PubSub[F]
 
   def withDelegate(delegate: PubSub[F]): PubSubT[F]
 }
 
 object PubSubT {
+  def noop[F[_]: Async]: PubSubT[F] =
+    PubSubT(PubSub.noop[F], Logger.off[F])
 
-  def apply[F[_]](pubSub: PubSub[F], logger: Logger[F]): PubSubT[F] =
+  def apply[F[_]: Async](pubSub: PubSub[F], logger: Logger[F]): PubSubT[F] =
     new PubSubT[F] {
       def publish1[A](topic: TypedTopic[A], msg: A): F[MessageHead] =
         pubSub.publish1(topic.topic, topic.codec(msg))
 
+      def publish1IgnoreErrors[A](topic: TypedTopic[A], msg: A): F[Unit] =
+        publish1(topic, msg).attempt.flatMap {
+          case Right(_) => ().pure[F]
+          case Left(ex) =>
+            logger.error(ex)(s"Error publishing to topic ${topic.topic.name}: $msg")
+        }
+
       def publish[A](topic: TypedTopic[A]): Pipe[F, A, MessageHead] =
         _.map(topic.codec.apply).through(pubSub.publish(topic.topic))
 
@@ -49,6 +65,18 @@ object PubSubT {
             }
           )
 
+      def subscribeSink[A](
+          topic: TypedTopic[A]
+      )(handler: Message[A] => F[Unit]): F[F[Unit]] =
+        for {
+          halt <- SignallingRef.of[F, Boolean](false)
+          _ <- subscribe(topic)
+            .evalMap(handler)
+            .interruptWhen(halt)
+            .compile
+            .drain
+        } yield halt.set(true)
+
       def delegate: PubSub[F] = pubSub
 
       def withDelegate(newDelegate: PubSub[F]): PubSubT[F] =

modules/pubsub/api/src/main/scala/docspell/pubsub/api/Topics.scala

@@ -1,61 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.pubsub.api
-
-import cats.data.NonEmptyList
-
-import docspell.common.Ident
-
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import io.circe.{Decoder, Encoder}
-
-/** All topics used in Docspell. */
-object Topics {
-
-  /** Notify when a job has finished. */
-  val jobDone: TypedTopic[JobDoneMsg] = TypedTopic[JobDoneMsg](Topic("job-done"))
-
-  /** Notify when a job has been submitted. The job executor listens to these messages to
-    * wake up and do its work.
-    */
-  val jobSubmitted: TypedTopic[JobSubmittedMsg] =
-    TypedTopic[JobSubmittedMsg](Topic("job-submitted"))
-
-  /** Notify a node to cancel a job with the given id */
-  val cancelJob: TypedTopic[CancelJobMsg] =
-    TypedTopic[CancelJobMsg](Topic("cancel-job"))
-
-  val all: NonEmptyList[TypedTopic[_]] = NonEmptyList.of(jobDone, jobSubmitted, cancelJob)
-
-  final case class JobSubmittedMsg(task: Ident)
-  object JobSubmittedMsg {
-    implicit val jsonDecoder: Decoder[JobSubmittedMsg] =
-      deriveDecoder[JobSubmittedMsg]
-
-    implicit val jsonEncoder: Encoder[JobSubmittedMsg] =
-      deriveEncoder[JobSubmittedMsg]
-  }
-
-  final case class JobDoneMsg(jobId: Ident, task: Ident)
-  object JobDoneMsg {
-    implicit val jsonDecoder: Decoder[JobDoneMsg] =
-      deriveDecoder[JobDoneMsg]
-
-    implicit val jsonEncoder: Encoder[JobDoneMsg] =
-      deriveEncoder[JobDoneMsg]
-  }
-
-  final case class CancelJobMsg(jobId: Ident, nodeId: Ident)
-  object CancelJobMsg {
-    implicit val jsonDecoder: Decoder[CancelJobMsg] =
-      deriveDecoder[CancelJobMsg]
-
-    implicit val jsonEncoder: Encoder[CancelJobMsg] =
-      deriveEncoder[CancelJobMsg]
-
-  }
-}

modules/pubsub/naive/src/main/scala/docspell/pubsub/naive/NaivePubSub.scala

@@ -155,23 +155,16 @@ final class NaivePubSub[F[_]: Async](
 
     for {
       _ <- logger.trace(s"Find all nodes subscribed to topic ${msg.head.topic.name}")
-      urls <- store.transact(RPubSub.findSubs(msg.head.topic.name))
+      urls <- store.transact(RPubSub.findSubs(msg.head.topic.name, cfg.nodeId))
       _ <- logger.trace(s"Publishing to remote urls ${urls.map(_.asString)}: $msg")
       reqs = urls
         .map(u => Uri.unsafeFromString(u.asString))
         .map(uri => POST(List(msg), uri))
-      res <- reqs.traverse(req => client.status(req)).attempt
-      _ <- res match {
+      resList <- reqs.traverse(req => client.status(req).attempt)
+      _ <- resList.traverse {
         case Right(s) =>
-          if (s.forall(_.isSuccess)) ().pure[F]
-          else if (s.size == urls.size)
-            logger.warn(
-              s"No nodes was be reached! Reason: $s, message: $msg"
-            )
-          else
-            logger.warn(
-              s"Some nodes were not reached! Reason: $s, message: $msg"
-            )
+          if (s.isSuccess) ().pure[F]
+          else logger.warn(s"A node was not reached! Reason: $s, message: $msg")
         case Left(ex) =>
           logger.error(ex)(s"Error publishing ${msg.head.topic.name} message remotely")
       }

modules/pubsub/naive/src/test/scala/docspell/pubsub/naive/NaivePubSubTest.scala

@@ -13,8 +13,8 @@ import cats.implicits._
 import fs2.concurrent.SignallingRef
 
 import docspell.common._
-import docspell.pubsub.api.Topics.{JobDoneMsg, JobSubmittedMsg}
 import docspell.pubsub.api._
+import docspell.pubsub.naive.Topics._
 
 import munit.CatsEffectSuite
 
@@ -104,7 +104,7 @@ class NaivePubSubTest extends CatsEffectSuite with Fixtures {
   }
 
   pubsubEnv.test("do not receive remote message from other topic") { env =>
-    val msg = JobDoneMsg("job-1".id, "task-2".id)
+    val msg = JobCancelMsg("job-1".id)
 
     // Create two pubsub instances connected to the same database
     conntectedPubsubs(env).use { case (ps1, ps2) =>
@@ -112,7 +112,7 @@ class NaivePubSubTest extends CatsEffectSuite with Fixtures {
         // subscribe to ps1 and send via ps2
         res <- subscribe(ps1, Topics.jobSubmitted)
         (received, halt, subFiber) = res
-        _ <- ps2.publish1(Topics.jobDone, msg)
+        _ <- ps2.publish1(Topics.jobCancel, msg)
         _ <- IO.sleep(100.millis)
         _ <- halt.set(true)
         outcome <- subFiber.join

modules/pubsub/naive/src/test/scala/docspell/pubsub/naive/Topics.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.pubsub.naive
+
+import cats.data.NonEmptyList
+
+import docspell.common.Ident
+import docspell.pubsub.api._
+
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import io.circe.{Decoder, Encoder}
+
+object Topics {
+  val jobSubmitted: TypedTopic[JobSubmittedMsg] =
+    TypedTopic[JobSubmittedMsg](Topic("test-job-submitted"))
+
+  final case class JobSubmittedMsg(task: Ident)
+  object JobSubmittedMsg {
+    implicit val encode: Encoder[JobSubmittedMsg] = deriveEncoder[JobSubmittedMsg]
+    implicit val decode: Decoder[JobSubmittedMsg] = deriveDecoder[JobSubmittedMsg]
+  }
+
+  val jobCancel: TypedTopic[JobCancelMsg] =
+    TypedTopic[JobCancelMsg](Topic("test-job-done"))
+  final case class JobCancelMsg(id: Ident)
+  object JobCancelMsg {
+    implicit val encode: Encoder[JobCancelMsg] = deriveEncoder[JobCancelMsg]
+    implicit val decode: Decoder[JobCancelMsg] = deriveDecoder[JobCancelMsg]
+  }
+
+  def all: NonEmptyList[TypedTopic[_]] =
+    NonEmptyList.of(jobSubmitted, jobCancel)
+}