Amend source form with tags and file-filter

Allow to define tags and a file filter per source.

modules/store/src/main/resources/db/migration/h2/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/resources/db/migration/mariadb/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE `source`
+ADD COLUMN `file_filter` varchar(254) NULL;
+
+CREATE TABLE `tagsource` (
+  `id` varchar(254) not null primary key,
+  `source_id` varchar(254) not null,
+  `tag_id` varchar(254) not null,
+  unique (`source_id`, `tag_id`),
+  foreign key (`source_id`) references `source`(`sid`),
+  foreign key (`tag_id`) references `tag`(`tid`)
+);

modules/store/src/main/resources/db/migration/postgresql/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala

@@ -91,6 +91,9 @@ trait DoobieMeta extends EmilDoobieMeta {
 
   implicit val metaCalEvent: Meta[CalEvent] =
     Meta[String].timap(CalEvent.unsafe)(_.asString)
+
+  implicit val metaGlob: Meta[Glob] =
+    Meta[String].timap(Glob.apply)(_.asString)
 }
 
 object DoobieMeta extends DoobieMeta {

modules/store/src/main/scala/docspell/store/records/RSource.scala

@@ -16,8 +16,13 @@ case class RSource(
     enabled: Boolean,
     priority: Priority,
     created: Timestamp,
-    folderId: Option[Ident]
-) {}
+    folderId: Option[Ident],
+    fileFilter: Option[Glob]
+) {
+
+  def fileFilterOrAll: Glob =
+    fileFilter.getOrElse(Glob.all)
+}
 
 object RSource {
 
@@ -34,9 +39,21 @@ object RSource {
     val priority    = Column("priority")
     val created     = Column("created")
     val folder      = Column("folder_id")
+    val fileFilter  = Column("file_filter")
 
     val all =
-      List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
+      List(
+        sid,
+        cid,
+        abbrev,
+        description,
+        counter,
+        enabled,
+        priority,
+        created,
+        folder,
+        fileFilter
+      )
   }
 
   import Columns._
@@ -45,7 +62,7 @@ object RSource {
     val sql = insertRow(
       table,
       all,
-      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
+      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}"
     )
     sql.update.run
   }
@@ -60,7 +77,8 @@ object RSource {
         description.setTo(v.description),
         enabled.setTo(v.enabled),
         priority.setTo(v.priority),
-        folder.setTo(v.folderId)
+        folder.setTo(v.folderId),
+        fileFilter.setTo(v.fileFilter)
       )
     )
     sql.update.run
@@ -83,10 +101,11 @@ object RSource {
     sql.query[Int].unique.map(_ > 0)
   }
 
-  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] = {
-    val sql = selectSimple(all, table, and(sid.is(id), enabled.is(true)))
-    sql.query[RSource].option
-  }
+  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] =
+    findEnabledSql(id).query[RSource].option
+
+  private[records] def findEnabledSql(id: Ident): Fragment =
+    selectSimple(all, table, and(sid.is(id), enabled.is(true)))
 
   def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] =
     selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option
@@ -94,10 +113,11 @@ object RSource {
   def findAll(
       coll: Ident,
       order: Columns.type => Column
-  ): ConnectionIO[Vector[RSource]] = {
-    val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
-    sql.query[RSource].to[Vector]
-  }
+  ): ConnectionIO[Vector[RSource]] =
+    findAllSql(coll, order).query[RSource].to[Vector]
+
+  private[records] def findAllSql(coll: Ident, order: Columns.type => Column): Fragment =
+    selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
 
   def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
     deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run

modules/store/src/main/scala/docspell/store/records/RTag.scala

@@ -104,6 +104,18 @@ object RTag {
     ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
   }
 
+  def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = {
+    val rcol = all.map(_.prefix("t"))
+    (selectSimple(
+      rcol,
+      table ++ fr"t," ++ RTagSource.table ++ fr"s",
+      and(
+        RTagSource.Columns.sourceId.prefix("s").is(source),
+        RTagSource.Columns.tagId.prefix("s").is(tid.prefix("t"))
+      )
+    ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
+  }
+
   def findAllByNameOrId(
       nameOrIds: List[String],
       coll: Ident

modules/store/src/main/scala/docspell/store/records/RTagSource.scala

@@ -0,0 +1,56 @@
+package docspell.store.records
+
+import cats.effect.Sync
+import cats.implicits._
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+case class RTagSource(id: Ident, sourceId: Ident, tagId: Ident) {}
+
+object RTagSource {
+
+  val table = fr"tagsource"
+
+  object Columns {
+    val id       = Column("id")
+    val sourceId = Column("source_id")
+    val tagId    = Column("tag_id")
+    val all      = List(id, sourceId, tagId)
+  }
+  import Columns._
+
+  def createNew[F[_]: Sync](source: Ident, tag: Ident): F[RTagSource] =
+    Ident.randomId[F].map(id => RTagSource(id, source, tag))
+
+  def insert(v: RTagSource): ConnectionIO[Int] =
+    insertRow(table, all, fr"${v.id},${v.sourceId},${v.tagId}").update.run
+
+  def deleteSourceTags(source: Ident): ConnectionIO[Int] =
+    deleteFrom(table, sourceId.is(source)).update.run
+
+  def deleteTag(tid: Ident): ConnectionIO[Int] =
+    deleteFrom(table, tagId.is(tid)).update.run
+
+  def findBySource(source: Ident): ConnectionIO[Vector[RTagSource]] =
+    selectSimple(all, table, sourceId.is(source)).query[RTagSource].to[Vector]
+
+  def setAllTags(source: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
+    if (tags.isEmpty) 0.pure[ConnectionIO]
+    else
+      for {
+        entities <- tags.toList.traverse(tagId =>
+          Ident.randomId[ConnectionIO].map(id => RTagSource(id, source, tagId))
+        )
+        n <- insertRows(
+          table,
+          all,
+          entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}")
+        ).update.run
+      } yield n
+
+}

modules/store/src/main/scala/docspell/store/records/SourceData.scala

@@ -0,0 +1,87 @@
+package docspell.store.records
+
+import cats.effect.concurrent.Ref
+import cats.implicits._
+import fs2.Stream
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+/** Combines a source record (RSource) and a list of associated tags.
+  */
+case class SourceData(source: RSource, tags: Vector[RTag])
+
+object SourceData {
+
+  def fromSource(s: RSource): SourceData =
+    SourceData(s, Vector.empty)
+
+  def findAll(
+      coll: Ident,
+      order: RSource.Columns.type => Column
+  ): Stream[ConnectionIO, SourceData] =
+    findAllWithTags(RSource.findAllSql(coll, order).query[RSource].stream)
+
+  private def findAllWithTags(
+      select: Stream[ConnectionIO, RSource]
+  ): Stream[ConnectionIO, SourceData] = {
+    def findTag(
+        cache: Ref[ConnectionIO, Map[Ident, RTag]],
+        tagSource: RTagSource
+    ): ConnectionIO[Option[RTag]] =
+      for {
+        cc <- cache.get
+        fromCache = cc.get(tagSource.tagId)
+        orFromDB <-
+          if (fromCache.isDefined) fromCache.pure[ConnectionIO]
+          else RTag.findById(tagSource.tagId)
+        _ <-
+          if (fromCache.isDefined) ().pure[ConnectionIO]
+          else
+            orFromDB match {
+              case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
+              case None    => ().pure[ConnectionIO]
+            }
+      } yield orFromDB
+
+    for {
+      resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
+      source       <- select
+      tagSources   <- Stream.eval(RTagSource.findBySource(source.sid))
+      tags         <- Stream.eval(tagSources.traverse(ti => findTag(resolvedTags, ti)))
+    } yield SourceData(source, tags.flatten)
+  }
+
+  def findEnabled(id: Ident): ConnectionIO[Option[SourceData]] =
+    findAllWithTags(RSource.findEnabledSql(id).query[RSource].stream).head.compile.last
+
+  def insert(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.insert(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def update(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.updateNoCounter(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      _    <- RTagSource.deleteSourceTags(data.sid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def delete(source: Ident, coll: Ident): ConnectionIO[Int] =
+    for {
+      n0 <- RTagSource.deleteSourceTags(source)
+      n1 <- RSource.delete(source, coll)
+    } yield n0 + n1
+
+}