Cleanup nodes that are not reachable anymore

modules/joex/src/main/resources/reference.conf

@@ -170,6 +170,14 @@ docspell.joex {
       # whether more or less memory should be used.
       delete-batch = "100"
     }
+
+    # Removes node entries that are not reachable anymore.
+    check-nodes {
+      # Whether this task is enabled
+      enabled = true
+      # How often the node must be unreachable, before it is removed.
+      min-not-found = 2
+    }
   }
 
   # Configuration of text extraction

modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala

@@ -0,0 +1,67 @@
+package docspell.joex.hk
+
+import cats.effect._
+import cats.implicits._
+
+import docspell.common._
+import docspell.joex.scheduler.{Context, Task}
+import docspell.store.records._
+
+import org.http4s.client.Client
+import org.http4s.client.blaze.BlazeClientBuilder
+
+object CheckNodesTask {
+
+  def apply[F[_]: ConcurrentEffect](
+      cfg: HouseKeepingConfig.CheckNodes
+  ): Task[F, Unit, Unit] =
+    Task { ctx =>
+      if (cfg.enabled)
+        for {
+          _ <- ctx.logger.info("Check nodes reachability")
+          _ <- BlazeClientBuilder[F](ctx.blocker.blockingContext).resource.use { client =>
+            checkNodes(ctx, client)
+          }
+          _ <- ctx.logger.info(
+            s"Remove nodes not found more than ${cfg.minNotFound} times"
+          )
+          n <- removeNodes(ctx, cfg)
+          _ <- ctx.logger.info(s"Removed $n nodes")
+        } yield ()
+      else
+        ctx.logger.info("CheckNodes task is disabled in the configuration")
+    }
+
+  def checkNodes[F[_]: Sync](ctx: Context[F, _], client: Client[F]): F[Unit] =
+    ctx.store
+      .transact(RNode.streamAll)
+      .evalMap(node =>
+        checkNode(ctx.logger, client)(node.url)
+          .flatMap(seen =>
+            if (seen) ctx.store.transact(RNode.resetNotFound(node.id))
+            else ctx.store.transact(RNode.incrementNotFound(node.id))
+          )
+      )
+      .compile
+      .drain
+
+  def checkNode[F[_]: Sync](logger: Logger[F], client: Client[F])(
+      url: LenientUri
+  ): F[Boolean] = {
+    val apiVersion = url / "api" / "info" / "version"
+    for {
+      res <- client.expect[String](apiVersion.asString).attempt
+      _ <- res.fold(
+        ex => logger.info(s"Node ${url.asString} not found: ${ex.getMessage}"),
+        _ => logger.info(s"Node ${url.asString} is reachable")
+      )
+    } yield res.isRight
+  }
+
+  def removeNodes[F[_]: Sync](
+      ctx: Context[F, _],
+      cfg: HouseKeepingConfig.CheckNodes
+  ): F[Int] =
+    ctx.store.transact(RNode.deleteNotFound(cfg.minNotFound))
+
+}

modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingConfig.scala

@@ -9,7 +9,8 @@ case class HouseKeepingConfig(
     schedule: CalEvent,
     cleanupInvites: CleanupInvites,
     cleanupJobs: CleanupJobs,
-    cleanupRememberMe: CleanupRememberMe
+    cleanupRememberMe: CleanupRememberMe,
+    checkNodes: CheckNodes
 )
 
 object HouseKeepingConfig {
@@ -20,4 +21,6 @@ object HouseKeepingConfig {
 
   case class CleanupRememberMe(enabled: Boolean, olderThan: Duration)
 
+  case class CheckNodes(enabled: Boolean, minNotFound: Int)
+
 }

modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala

@@ -15,12 +15,13 @@ object HouseKeepingTask {
 
   val taskName: Ident = Ident.unsafe("housekeeping")
 
-  def apply[F[_]: Sync](cfg: Config): Task[F, Unit, Unit] =
+  def apply[F[_]: ConcurrentEffect](cfg: Config): Task[F, Unit, Unit] =
     Task
       .log[F, Unit](_.info(s"Running house-keeping task now"))
       .flatMap(_ => CleanupInvitesTask(cfg.houseKeeping.cleanupInvites))
       .flatMap(_ => CleanupRememberMeTask(cfg.houseKeeping.cleanupRememberMe))
       .flatMap(_ => CleanupJobsTask(cfg.houseKeeping.cleanupJobs))
+      .flatMap(_ => CheckNodesTask(cfg.houseKeeping.checkNodes))
 
   def onCancel[F[_]: Sync]: Task[F, Unit, Unit] =
     Task.log[F, Unit](_.warn("Cancelling house-keeping task"))