Cleanup nodes that are not reachable anymore

modules/store/src/main/resources/db/migration/h2/V1.20.3__node_seen.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "node"
+ADD COLUMN "not_found" int not null default 0;

modules/store/src/main/resources/db/migration/mariadb/V1.20.3__node_seen.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `node`
+ADD COLUMN `not_found` int not null default 0;

modules/store/src/main/resources/db/migration/postgresql/V1.20.3__node_seen.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "node"
+ADD COLUMN "not_found" int not null default 0;

modules/store/src/main/scala/docspell/store/records/RNode.scala

@@ -1,8 +1,8 @@
 package docspell.store.records
-
 import cats.data.NonEmptyList
 import cats.effect.Sync
 import cats.implicits._
+import fs2.Stream
 
 import docspell.common._
 import docspell.store.qb.DSL._
@@ -16,13 +16,14 @@ case class RNode(
     nodeType: NodeType,
     url: LenientUri,
     updated: Timestamp,
-    created: Timestamp
+    created: Timestamp,
+    notFound: Int
 ) {}
 
 object RNode {
 
   def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] =
-    Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now))
+    Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now, 0))
 
   final case class Table(alias: Option[String]) extends TableDef {
     val tableName = "node"
@@ -32,18 +33,20 @@ object RNode {
     val url      = Column[LenientUri]("url", this)
     val updated  = Column[Timestamp]("updated", this)
     val created  = Column[Timestamp]("created", this)
-    val all      = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created)
+    val notFound = Column[Int]("not_found", this)
+    val all      = NonEmptyList.of[Column[_]](id, nodeType, url, updated, created, notFound)
   }
 
   def as(alias: String): Table =
     Table(Some(alias))
+  val T = Table(None)
 
   def insert(v: RNode): ConnectionIO[Int] = {
     val t = Table(None)
     DML.insert(
       t,
       t.all,
-      fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}"
+      fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created},${v.notFound}"
     )
   }
 
@@ -61,6 +64,22 @@ object RNode {
       )
   }
 
+  def incrementNotFound(nid: Ident): ConnectionIO[Int] =
+    Timestamp
+      .current[ConnectionIO]
+      .flatMap(now =>
+        DML
+          .update(T, T.id === nid, DML.set(T.notFound.increment(1), T.updated.setTo(now)))
+      )
+
+  def resetNotFound(id: Ident): ConnectionIO[Int] =
+    Timestamp
+      .current[ConnectionIO]
+      .flatMap(now =>
+        DML
+          .update(T, T.id === id, DML.set(T.notFound.setTo(0), T.updated.setTo(now)))
+      )
+
   def set(v: RNode): ConnectionIO[Int] =
     for {
       n <- update(v)
@@ -81,4 +100,10 @@ object RNode {
     val t = Table(None)
     run(select(t.all), from(t), t.id === nodeId).query[RNode].option
   }
+
+  def streamAll: Stream[ConnectionIO, RNode] =
+    run(select(T.all), from(T)).query[RNode].streamWithChunkSize(50)
+
+  def deleteNotFound(min: Int): ConnectionIO[Int] =
+    DML.delete(T, T.notFound >= min)
 }