Add websockets and notify frontend when an item is processed

modules/restserver/src/main/scala/docspell/restserver/Main.scala

@@ -38,12 +38,6 @@ object Main extends IOApp {
 
     pools = connectEC.map(Pools.apply)
     rc <-
-      pools.use(p =>
-        RestServer
-          .stream[IO](cfg, p)
-          .compile
-          .drain
-          .as(ExitCode.Success)
-      )
+      pools.use(p => RestServer.serve[IO](cfg, p))
   } yield rc
 }

modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala

@@ -7,10 +7,8 @@
 package docspell.restserver
 
 import cats.effect._
-import cats.implicits._
 
 import docspell.backend.BackendApp
-import docspell.backend.msg.{JobDone, Ping}
 import docspell.common.Logger
 import docspell.ftsclient.FtsClient
 import docspell.ftssolr.SolrFtsClient
@@ -35,24 +33,10 @@ object RestAppImpl {
       ftsClient <- createFtsClient(cfg)(httpClient)
       pubSubT = PubSubT(pubSub, logger)
       backend <- BackendApp.create[F](cfg.backend, store, ftsClient, pubSubT)
-      _ <- Resource.eval(subscriptions(backend, logger))
       app = new RestAppImpl[F](cfg, backend)
     } yield app
   }
 
-  private def subscriptions[F[_]: Async](
-      backend: BackendApp[F],
-      logger: Logger[F]
-  ): F[Unit] =
-    for {
-      _ <- Async[F].start(backend.pubSub.subscribeSink(Ping.topic) { msg =>
-        logger.info(s">>>> PING $msg")
-      })
-      _ <- Async[F].start(backend.pubSub.subscribeSink(JobDone.topic) { msg =>
-        logger.info(s">>>> Job Done $msg")
-      })
-    } yield ()
-
   private def createFtsClient[F[_]: Async](
       cfg: Config
   )(client: Client[F]): Resource[F, FtsClient[F]] =

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -6,9 +6,12 @@
 
 package docspell.restserver
 
+import scala.concurrent.duration._
+
 import cats.effect._
 import cats.implicits._
 import fs2.Stream
+import fs2.concurrent.Topic
 
 import docspell.backend.auth.{AuthToken, ShareToken}
 import docspell.backend.msg.Topics
@@ -19,6 +22,8 @@ import docspell.restserver.auth.OpenId
 import docspell.restserver.http4s.EnvMiddleware
 import docspell.restserver.routes._
 import docspell.restserver.webapp._
+import docspell.restserver.ws.OutputEvent.KeepAlive
+import docspell.restserver.ws.{OutputEvent, WebSocketRoutes}
 import docspell.store.Store
 
 import org.http4s._
@@ -30,63 +35,96 @@ import org.http4s.headers.Location
 import org.http4s.implicits._
 import org.http4s.server.Router
 import org.http4s.server.middleware.Logger
+import org.http4s.server.websocket.WebSocketBuilder2
 
 object RestServer {
 
-  def stream[F[_]: Async](cfg: Config, pools: Pools): Stream[F, Nothing] = {
+  def serve[F[_]: Async](cfg: Config, pools: Pools): F[ExitCode] =
+    for {
+      wsTopic <- Topic[F, OutputEvent]
+      keepAlive = Stream
+        .awakeEvery[F](30.seconds)
+        .map(_ => KeepAlive)
+        .through(wsTopic.publish)
 
-    val templates = TemplateRoutes[F](cfg)
-    val app = for {
+      server =
+        Stream
+          .resource(createApp(cfg, pools))
+          .flatMap { case (restApp, pubSub, httpClient) =>
+            Stream(
+              Subscriptions(wsTopic, restApp.backend.pubSub),
+              BlazeServerBuilder[F]
+                .bindHttp(cfg.bind.port, cfg.bind.address)
+                .withoutBanner
+                .withHttpWebSocketApp(
+                  createHttpApp(cfg, httpClient, pubSub, restApp, wsTopic)
+                )
+                .serve
+                .drain
+            )
+          }
+
+      exit <-
+        (server ++ Stream(keepAlive)).parJoinUnbounded.compile.drain.as(ExitCode.Success)
+    } yield exit
+
+  def createApp[F[_]: Async](
+      cfg: Config,
+      pools: Pools
+  ): Resource[F, (RestApp[F], NaivePubSub[F], Client[F])] =
+    for {
+      httpClient <- BlazeClientBuilder[F].resource
       store <- Store.create[F](
         cfg.backend.jdbc,
         cfg.backend.files.chunkSize,
         pools.connectEC
       )
-      httpClient <- BlazeClientBuilder[F].resource
       pubSub <- NaivePubSub(cfg.pubSubConfig, store, httpClient)(Topics.all.map(_.topic))
       restApp <- RestAppImpl.create[F](cfg, store, httpClient, pubSub)
-      httpClient <- BlazeClientBuilder[F].resource
-      httpApp = Router(
-        "/internal/pubsub" -> pubSub.receiveRoute,
-        "/api/info" -> routes.InfoRoutes(),
-        "/api/v1/open/" -> openRoutes(cfg, httpClient, restApp),
-        "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
-          securedRoutes(cfg, restApp, token)
-        },
-        "/api/v1/admin" -> AdminAuth(cfg.adminEndpoint) {
-          adminRoutes(cfg, restApp)
-        },
-        "/api/v1/share" -> ShareAuth(restApp.backend.share, cfg.auth) { token =>
-          shareRoutes(cfg, restApp, token)
-        },
-        "/api/doc" -> templates.doc,
-        "/app/assets" -> EnvMiddleware(WebjarRoutes.appRoutes[F]),
-        "/app" -> EnvMiddleware(templates.app),
-        "/sw.js" -> EnvMiddleware(templates.serviceWorker),
-        "/" -> redirectTo("/app")
-      ).orNotFound
+    } yield (restApp, pubSub, httpClient)
 
-      finalHttpApp = Logger.httpApp(logHeaders = false, logBody = false)(httpApp)
+  def createHttpApp[F[_]: Async](
+      cfg: Config,
+      httpClient: Client[F],
+      pubSub: NaivePubSub[F],
+      restApp: RestApp[F],
+      topic: Topic[F, OutputEvent]
+  )(
+      wsB: WebSocketBuilder2[F]
+  ) = {
+    val templates = TemplateRoutes[F](cfg)
+    val httpApp = Router(
+      "/internal/pubsub" -> pubSub.receiveRoute,
+      "/api/info" -> routes.InfoRoutes(),
+      "/api/v1/open/" -> openRoutes(cfg, httpClient, restApp),
+      "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token =>
+        securedRoutes(cfg, restApp, wsB, topic, token)
+      },
+      "/api/v1/admin" -> AdminAuth(cfg.adminEndpoint) {
+        adminRoutes(cfg, restApp)
+      },
+      "/api/v1/share" -> ShareAuth(restApp.backend.share, cfg.auth) { token =>
+        shareRoutes(cfg, restApp, token)
+      },
+      "/api/doc" -> templates.doc,
+      "/app/assets" -> EnvMiddleware(WebjarRoutes.appRoutes[F]),
+      "/app" -> EnvMiddleware(templates.app),
+      "/sw.js" -> EnvMiddleware(templates.serviceWorker),
+      "/" -> redirectTo("/app")
+    ).orNotFound
 
-    } yield finalHttpApp
-
-    Stream
-      .resource(app)
-      .flatMap(httpApp =>
-        BlazeServerBuilder[F]
-          .bindHttp(cfg.bind.port, cfg.bind.address)
-          .withHttpApp(httpApp)
-          .withoutBanner
-          .serve
-      )
-  }.drain
+    Logger.httpApp(logHeaders = false, logBody = false)(httpApp)
+  }
 
   def securedRoutes[F[_]: Async](
       cfg: Config,
       restApp: RestApp[F],
+      wsB: WebSocketBuilder2[F],
+      topic: Topic[F, OutputEvent],
       token: AuthToken
   ): HttpRoutes[F] =
     Router(
+      "ws" -> WebSocketRoutes(token, topic, wsB),
       "auth" -> LoginRoutes.session(restApp.backend.login, cfg, token),
       "tag" -> TagRoutes(restApp.backend, token),
       "equipment" -> EquipmentRoutes(restApp.backend, token),

modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver
+
+import fs2.Stream
+import fs2.concurrent.Topic
+
+import docspell.backend.msg.JobDone
+import docspell.common._
+import docspell.common.syntax.StringSyntax._
+import docspell.pubsub.api.PubSubT
+import docspell.restserver.ws.OutputEvent
+
+/** Subscribes to those events from docspell that are forwarded to the websocket endpoints
+  */
+object Subscriptions {
+
+  def apply[F[_]](
+      wsTopic: Topic[F, OutputEvent],
+      pubSub: PubSubT[F]
+  ): Stream[F, Nothing] =
+    jobDone(pubSub).through(wsTopic.publish)
+
+  def jobDone[F[_]](pubSub: PubSubT[F]): Stream[F, OutputEvent] =
+    pubSub
+      .subscribe(JobDone.topic)
+      .filter(m => m.body.task == ProcessItemArgs.taskName)
+      .map(m => m.body.args.parseJsonAs[ProcessItemArgs])
+      .collect { case Right(a) => OutputEvent.ItemProcessed(a.meta.collective) }
+
+}

modules/restserver/src/main/scala/docspell/restserver/ws/InputMessage.scala

@@ -0,0 +1,11 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver.ws
+
+sealed trait InputMessage
+
+object InputMessage {}

modules/restserver/src/main/scala/docspell/restserver/ws/OutputEvent.scala

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver.ws
+
+import docspell.backend.auth.AuthToken
+import docspell.common._
+
+sealed trait OutputEvent {
+  def forCollective(token: AuthToken): Boolean
+  def encode: String
+}
+
+object OutputEvent {
+
+  case object KeepAlive extends OutputEvent {
+    def forCollective(token: AuthToken): Boolean = true
+    def encode: String = "keep-alive"
+  }
+
+  final case class ItemProcessed(collective: Ident) extends OutputEvent {
+    def forCollective(token: AuthToken): Boolean =
+      token.account.collective == collective
+
+    def encode: String =
+      "item-processed"
+  }
+
+}

modules/restserver/src/main/scala/docspell/restserver/ws/WebSocketRoutes.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.restserver.ws
+
+import cats.effect.Async
+import fs2.concurrent.Topic
+import fs2.{Pipe, Stream}
+
+import docspell.backend.auth.AuthToken
+
+import org.http4s.HttpRoutes
+import org.http4s.dsl.Http4sDsl
+import org.http4s.server.websocket.WebSocketBuilder2
+import org.http4s.websocket.WebSocketFrame
+import org.http4s.websocket.WebSocketFrame.Text
+
+object WebSocketRoutes {
+
+  def apply[F[_]: Async](
+      user: AuthToken,
+      topic: Topic[F, OutputEvent],
+      wsb: WebSocketBuilder2[F]
+  ): HttpRoutes[F] = {
+    val dsl = new Http4sDsl[F] {}
+    import dsl._
+
+    HttpRoutes.of { case GET -> Root =>
+      val toClient: Stream[F, WebSocketFrame.Text] =
+        topic
+          .subscribe(500)
+          .filter(_.forCollective(user))
+          .map(msg => Text(msg.encode))
+
+      val toServer: Pipe[F, WebSocketFrame, Unit] =
+        _.map(_ => ())
+
+      wsb.build(toClient, toServer)
+    }
+  }
+}