Amend source form with tags and file-filter

Allow to define tags and a file filter per source.
2025-10-31 17:50:11 +00:00 · 2020-11-12 21:40:53 +01:00
parent 4fd6e02ec0
commit 04ba14f802
18 changed files with 498 additions and 122 deletions
--- a/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala
@@ -4,44 +4,50 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._

 import docspell.common.{AccountId, Ident}
+import docspell.store.UpdateResult
 import docspell.store.records.RSource
+import docspell.store.records.SourceData
 import docspell.store.{AddResult, Store}

 trait OSource[F[_]] {

-  def findAll(account: AccountId): F[Vector[RSource]]
+  def findAll(account: AccountId): F[Vector[SourceData]]

-  def add(s: RSource): F[AddResult]
+  def add(s: RSource, tags: List[String]): F[AddResult]

-  def update(s: RSource): F[AddResult]
+  def update(s: RSource, tags: List[String]): F[AddResult]

-  def delete(id: Ident, collective: Ident): F[AddResult]
+  def delete(id: Ident, collective: Ident): F[UpdateResult]
 }

 object OSource {

  def apply[F[_]: Effect](store: Store[F]): Resource[F, OSource[F]] =
    Resource.pure[F, OSource[F]](new OSource[F] {
-      def findAll(account: AccountId): F[Vector[RSource]] =
-        store.transact(RSource.findAll(account.collective, _.abbrev))
+      def findAll(account: AccountId): F[Vector[SourceData]] =
+        store
+          .transact(SourceData.findAll(account.collective, _.abbrev))
+          .compile
+          .to(Vector)

-      def add(s: RSource): F[AddResult] = {
-        def insert = RSource.insert(s)
+      def add(s: RSource, tags: List[String]): F[AddResult] = {
+        def insert = SourceData.insert(s, tags)
        def exists = RSource.existsByAbbrev(s.cid, s.abbrev)

        val msg = s"A source with abbrev '${s.abbrev}' already exists"
        store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
      }

-      def update(s: RSource): F[AddResult] = {
-        def insert = RSource.updateNoCounter(s)
+      def update(s: RSource, tags: List[String]): F[AddResult] = {
+        def insert = SourceData.update(s, tags)
        def exists = RSource.existsByAbbrev(s.cid, s.abbrev)

        val msg = s"A source with abbrev '${s.abbrev}' already exists"
        store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
      }

-      def delete(id: Ident, collective: Ident): F[AddResult] =
-        store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
+      def delete(id: Ident, collective: Ident): F[UpdateResult] =
+        UpdateResult.fromUpdate(store.transact(SourceData.delete(id, collective)))
+
    })
 }
--- a/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala
@@ -4,6 +4,7 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._

 import docspell.common.{AccountId, Ident}
+import docspell.store.records.RTagSource
 import docspell.store.records.{RTag, RTagItem}
 import docspell.store.{AddResult, Store}

@@ -49,8 +50,9 @@ object OTag {
        val io = for {
          optTag <- RTag.findByIdAndCollective(id, collective)
          n0     <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
-          n1     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
-        } yield n0.getOrElse(0) + n1.getOrElse(0)
+          n1     <- optTag.traverse(t => RTagSource.deleteTag(t.tagId))
+          n2     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
+        } yield (n0 |+| n1 |+| n2).getOrElse(0)
        store.transact(io).attempt.map(AddResult.fromUpdate)
      }

--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -25,6 +25,11 @@ trait OUpload[F[_]] {
      itemId: Option[Ident]
  ): F[OUpload.UploadResult]

+  /** Submit files via a given source identifier. The source is looked
+    * up to identify the collective the files belong to. Metadata
+    * defined in the source is used as a fallback to those specified
+    * here (in UploadData).
+    */
  def submit(
      data: OUpload.UploadData[F],
      sourceId: Ident,
@@ -153,15 +158,19 @@ object OUpload {
          itemId: Option[Ident]
      ): F[OUpload.UploadResult] =
        (for {
-          src <- OptionT(store.transact(RSource.findEnabled(sourceId)))
+          src <- OptionT(store.transact(SourceData.findEnabled(sourceId)))
          updata = data.copy(
            meta = data.meta.copy(
-              sourceAbbrev = src.abbrev,
-              folderId = data.meta.folderId.orElse(src.folderId)
+              sourceAbbrev = src.source.abbrev,
+              folderId = data.meta.folderId.orElse(src.source.folderId),
+              fileFilter =
+                if (data.meta.fileFilter == Glob.all) src.source.fileFilterOrAll
+                else data.meta.fileFilter,
+              tags = (data.meta.tags ++ src.tags.map(_.tagId.id)).distinct
            ),
-            priority = src.priority
+            priority = src.source.priority
          )
-          accId = AccountId(src.cid, src.sid)
+          accId = AccountId(src.source.cid, src.source.sid)
          result <- OptionT.liftF(submit(updata, accId, notifyJoex, itemId))
        } yield result).getOrElse(UploadResult.noSource)

--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -1211,7 +1211,7 @@ paths:
        content:
          application/json:
            schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
      responses:
        200:
          description: Ok
@@ -1231,7 +1231,7 @@ paths:
        content:
          application/json:
            schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
      responses:
        200:
          description: Ok
@@ -4366,7 +4366,7 @@ components:
        items:
          type: array
          items:
-            $ref: "#/components/schemas/Source"
+            $ref: "#/components/schemas/SourceAndTags"
    Source:
      description: |
        Data about a Source. A source defines the endpoint where
@@ -4400,10 +4400,38 @@ components:
        folder:
          type: string
          format: ident
+        fileFilter:
+          type: string
+          format: glob
        created:
          description: DateTime
          type: integer
          format: date-time
+    SourceTagIn:
+      description: |
+        A source and optional tags (ids or names) for updating/adding.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          type: array
+          items:
+            type: string
+    SourceAndTags:
+      description: |
+        A source and optional tags.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          $ref: "#/components/schema/TagList"
+
    EquipmentList:
      description: |
        A list of equipments.
--- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@@ -524,21 +524,36 @@ trait Conversions {

  // sources

-  def mkSource(s: RSource): Source =
-    Source(
-      s.sid,
-      s.abbrev,
-      s.description,
-      s.counter,
-      s.enabled,
-      s.priority,
-      s.folderId,
-      s.created
+  def mkSource(s: SourceData): SourceAndTags =
+    SourceAndTags(
+      Source(
+        s.source.sid,
+        s.source.abbrev,
+        s.source.description,
+        s.source.counter,
+        s.source.enabled,
+        s.source.priority,
+        s.source.folderId,
+        s.source.fileFilter,
+        s.source.created
+      ),
+      TagList(s.tags.length, s.tags.map(mkTag).toList)
    )

  def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
    timeId.map({ case (id, now) =>
-      RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder)
+      RSource(
+        id,
+        cid,
+        s.abbrev,
+        s.description,
+        0,
+        s.enabled,
+        s.priority,
+        now,
+        s.folder,
+        s.fileFilter
+      )
    })

  def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
@@ -551,7 +566,8 @@ trait Conversions {
      s.enabled,
      s.priority,
      s.created,
-      s.folder
+      s.folder,
+      s.fileFilter
    )

  // equipment
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala
@@ -30,17 +30,17 @@ object SourceRoutes {

      case req @ POST -> Root =>
        for {
-          data  <- req.as[Source]
-          src   <- newSource(data, user.account.collective)
-          added <- backend.source.add(src)
+          data  <- req.as[SourceTagIn]
+          src   <- newSource(data.source, user.account.collective)
+          added <- backend.source.add(src, data.tags)
          resp  <- Ok(basicResult(added, "Source added."))
        } yield resp

      case req @ PUT -> Root =>
        for {
-          data <- req.as[Source]
-          src = changeSource(data, user.account.collective)
-          updated <- backend.source.update(src)
+          data <- req.as[SourceTagIn]
+          src = changeSource(data.source, user.account.collective)
+          updated <- backend.source.update(src, data.tags)
          resp    <- Ok(basicResult(updated, "Source updated."))
        } yield resp

--- a/modules/store/src/main/resources/db/migration/h2/V1.12.0__upload_data.sql
+++ b/modules/store/src/main/resources/db/migration/h2/V1.12.0__upload_data.sql
@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);
--- a/modules/store/src/main/resources/db/migration/mariadb/V1.12.0__upload_data.sql
+++ b/modules/store/src/main/resources/db/migration/mariadb/V1.12.0__upload_data.sql
@@ -0,0 +1,11 @@
+ALTER TABLE `source`
+ADD COLUMN `file_filter` varchar(254) NULL;
+
+CREATE TABLE `tagsource` (
+  `id` varchar(254) not null primary key,
+  `source_id` varchar(254) not null,
+  `tag_id` varchar(254) not null,
+  unique (`source_id`, `tag_id`),
+  foreign key (`source_id`) references `source`(`sid`),
+  foreign key (`tag_id`) references `tag`(`tid`)
+);
--- a/modules/store/src/main/resources/db/migration/postgresql/V1.12.0__upload_data.sql
+++ b/modules/store/src/main/resources/db/migration/postgresql/V1.12.0__upload_data.sql
@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);
--- a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala
+++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala
@@ -91,6 +91,9 @@ trait DoobieMeta extends EmilDoobieMeta {

  implicit val metaCalEvent: Meta[CalEvent] =
    Meta[String].timap(CalEvent.unsafe)(_.asString)
+
+  implicit val metaGlob: Meta[Glob] =
+    Meta[String].timap(Glob.apply)(_.asString)
 }

 object DoobieMeta extends DoobieMeta {
--- a/modules/store/src/main/scala/docspell/store/records/RSource.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala
@@ -16,8 +16,13 @@ case class RSource(
    enabled: Boolean,
    priority: Priority,
    created: Timestamp,
-    folderId: Option[Ident]
-) {}
+    folderId: Option[Ident],
+    fileFilter: Option[Glob]
+) {
+
+  def fileFilterOrAll: Glob =
+    fileFilter.getOrElse(Glob.all)
+}

 object RSource {

@@ -34,9 +39,21 @@ object RSource {
    val priority    = Column("priority")
    val created     = Column("created")
    val folder      = Column("folder_id")
+    val fileFilter  = Column("file_filter")

    val all =
-      List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
+      List(
+        sid,
+        cid,
+        abbrev,
+        description,
+        counter,
+        enabled,
+        priority,
+        created,
+        folder,
+        fileFilter
+      )
  }

  import Columns._
@@ -45,7 +62,7 @@ object RSource {
    val sql = insertRow(
      table,
      all,
-      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
+      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}"
    )
    sql.update.run
  }
@@ -60,7 +77,8 @@ object RSource {
        description.setTo(v.description),
        enabled.setTo(v.enabled),
        priority.setTo(v.priority),
-        folder.setTo(v.folderId)
+        folder.setTo(v.folderId),
+        fileFilter.setTo(v.fileFilter)
      )
    )
    sql.update.run
@@ -83,10 +101,11 @@ object RSource {
    sql.query[Int].unique.map(_ > 0)
  }

-  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] = {
-    val sql = selectSimple(all, table, and(sid.is(id), enabled.is(true)))
-    sql.query[RSource].option
-  }
+  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] =
+    findEnabledSql(id).query[RSource].option
+
+  private[records] def findEnabledSql(id: Ident): Fragment =
+    selectSimple(all, table, and(sid.is(id), enabled.is(true)))

  def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] =
    selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option
@@ -94,10 +113,11 @@ object RSource {
  def findAll(
      coll: Ident,
      order: Columns.type => Column
-  ): ConnectionIO[Vector[RSource]] = {
-    val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
-    sql.query[RSource].to[Vector]
-  }
+  ): ConnectionIO[Vector[RSource]] =
+    findAllSql(coll, order).query[RSource].to[Vector]
+
+  private[records] def findAllSql(coll: Ident, order: Columns.type => Column): Fragment =
+    selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)

  def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
    deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run
--- a/modules/store/src/main/scala/docspell/store/records/RTag.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala
@@ -104,6 +104,18 @@ object RTag {
    ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
  }

+  def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = {
+    val rcol = all.map(_.prefix("t"))
+    (selectSimple(
+      rcol,
+      table ++ fr"t," ++ RTagSource.table ++ fr"s",
+      and(
+        RTagSource.Columns.sourceId.prefix("s").is(source),
+        RTagSource.Columns.tagId.prefix("s").is(tid.prefix("t"))
+      )
+    ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
+  }
+
  def findAllByNameOrId(
      nameOrIds: List[String],
      coll: Ident
--- a/modules/store/src/main/scala/docspell/store/records/RTagSource.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RTagSource.scala
@@ -0,0 +1,56 @@
+package docspell.store.records
+
+import cats.effect.Sync
+import cats.implicits._
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+case class RTagSource(id: Ident, sourceId: Ident, tagId: Ident) {}
+
+object RTagSource {
+
+  val table = fr"tagsource"
+
+  object Columns {
+    val id       = Column("id")
+    val sourceId = Column("source_id")
+    val tagId    = Column("tag_id")
+    val all      = List(id, sourceId, tagId)
+  }
+  import Columns._
+
+  def createNew[F[_]: Sync](source: Ident, tag: Ident): F[RTagSource] =
+    Ident.randomId[F].map(id => RTagSource(id, source, tag))
+
+  def insert(v: RTagSource): ConnectionIO[Int] =
+    insertRow(table, all, fr"${v.id},${v.sourceId},${v.tagId}").update.run
+
+  def deleteSourceTags(source: Ident): ConnectionIO[Int] =
+    deleteFrom(table, sourceId.is(source)).update.run
+
+  def deleteTag(tid: Ident): ConnectionIO[Int] =
+    deleteFrom(table, tagId.is(tid)).update.run
+
+  def findBySource(source: Ident): ConnectionIO[Vector[RTagSource]] =
+    selectSimple(all, table, sourceId.is(source)).query[RTagSource].to[Vector]
+
+  def setAllTags(source: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
+    if (tags.isEmpty) 0.pure[ConnectionIO]
+    else
+      for {
+        entities <- tags.toList.traverse(tagId =>
+          Ident.randomId[ConnectionIO].map(id => RTagSource(id, source, tagId))
+        )
+        n <- insertRows(
+          table,
+          all,
+          entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}")
+        ).update.run
+      } yield n
+
+}
--- a/modules/store/src/main/scala/docspell/store/records/SourceData.scala
+++ b/modules/store/src/main/scala/docspell/store/records/SourceData.scala
@@ -0,0 +1,87 @@
+package docspell.store.records
+
+import cats.effect.concurrent.Ref
+import cats.implicits._
+import fs2.Stream
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+/** Combines a source record (RSource) and a list of associated tags.
+  */
+case class SourceData(source: RSource, tags: Vector[RTag])
+
+object SourceData {
+
+  def fromSource(s: RSource): SourceData =
+    SourceData(s, Vector.empty)
+
+  def findAll(
+      coll: Ident,
+      order: RSource.Columns.type => Column
+  ): Stream[ConnectionIO, SourceData] =
+    findAllWithTags(RSource.findAllSql(coll, order).query[RSource].stream)
+
+  private def findAllWithTags(
+      select: Stream[ConnectionIO, RSource]
+  ): Stream[ConnectionIO, SourceData] = {
+    def findTag(
+        cache: Ref[ConnectionIO, Map[Ident, RTag]],
+        tagSource: RTagSource
+    ): ConnectionIO[Option[RTag]] =
+      for {
+        cc <- cache.get
+        fromCache = cc.get(tagSource.tagId)
+        orFromDB <-
+          if (fromCache.isDefined) fromCache.pure[ConnectionIO]
+          else RTag.findById(tagSource.tagId)
+        _ <-
+          if (fromCache.isDefined) ().pure[ConnectionIO]
+          else
+            orFromDB match {
+              case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
+              case None    => ().pure[ConnectionIO]
+            }
+      } yield orFromDB
+
+    for {
+      resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
+      source       <- select
+      tagSources   <- Stream.eval(RTagSource.findBySource(source.sid))
+      tags         <- Stream.eval(tagSources.traverse(ti => findTag(resolvedTags, ti)))
+    } yield SourceData(source, tags.flatten)
+  }
+
+  def findEnabled(id: Ident): ConnectionIO[Option[SourceData]] =
+    findAllWithTags(RSource.findEnabledSql(id).query[RSource].stream).head.compile.last
+
+  def insert(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.insert(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def update(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.updateNoCounter(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      _    <- RTagSource.deleteSourceTags(data.sid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def delete(source: Ident, coll: Ident): ConnectionIO[Int] =
+    for {
+      n0 <- RTagSource.deleteSourceTags(source)
+      n1 <- RSource.delete(source, coll)
+    } yield n0 + n1
+
+}
--- a/modules/webapp/src/main/elm/Api.elm
+++ b/modules/webapp/src/main/elm/Api.elm
@@ -175,8 +175,9 @@ import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
 import Api.Model.ScanMailboxSettingsList exposing (ScanMailboxSettingsList)
 import Api.Model.SentMails exposing (SentMails)
 import Api.Model.SimpleMail exposing (SimpleMail)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Api.Model.StringList exposing (StringList)
 import Api.Model.Tag exposing (Tag)
 import Api.Model.TagCloud exposing (TagCloud)
@@ -1144,17 +1145,22 @@ getSources flags receive =
        }


-postSource : Flags -> Source -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+postSource : Flags -> SourceAndTags -> (Result Http.Error BasicResult -> msg) -> Cmd msg
 postSource flags source receive =
    let
+        st =
+            { source = source.source
+            , tags = List.map .id source.tags.items
+            }
+
        params =
            { url = flags.config.baseUrl ++ "/api/v1/sec/source"
            , account = getAccount flags
-            , body = Http.jsonBody (Api.Model.Source.encode source)
+            , body = Http.jsonBody (Api.Model.SourceTagIn.encode st)
            , expect = Http.expectJson receive Api.Model.BasicResult.decoder
            }
    in
-    if source.id == "" then
+    if source.source.id == "" then
        Http2.authPost params

    else
--- a/modules/webapp/src/main/elm/Comp/SourceForm.elm
+++ b/modules/webapp/src/main/elm/Comp/SourceForm.elm
@@ -12,7 +12,9 @@ import Api
 import Api.Model.FolderItem exposing (FolderItem)
 import Api.Model.FolderList exposing (FolderList)
 import Api.Model.IdName exposing (IdName)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
+import Api.Model.Tag exposing (Tag)
+import Api.Model.TagList exposing (TagList)
 import Comp.Dropdown exposing (isDropdownChangeMsg)
 import Comp.FixedDropdown
 import Data.Flags exposing (Flags)
@@ -24,10 +26,13 @@ import Html.Events exposing (onCheck, onInput)
 import Http
 import Markdown
 import Util.Folder exposing (mkFolderOption)
+import Util.Maybe
+import Util.Tag
+import Util.Update


 type alias Model =
-    { source : Source
+    { source : SourceAndTags
    , abbrev : String
    , description : Maybe String
    , priorityModel : Comp.FixedDropdown.Model Priority
@@ -36,12 +41,14 @@ type alias Model =
    , folderModel : Comp.Dropdown.Model IdName
    , allFolders : List FolderItem
    , folderId : Maybe String
+    , tagModel : Comp.Dropdown.Model Tag
+    , fileFilter : Maybe String
    }


 emptyModel : Model
 emptyModel =
-    { source = Api.Model.Source.empty
+    { source = Api.Model.SourceAndTags.empty
    , abbrev = ""
    , description = Nothing
    , priorityModel =
@@ -57,13 +64,18 @@ emptyModel =
            }
    , allFolders = []
    , folderId = Nothing
+    , tagModel = Util.Tag.makeDropdownModel
+    , fileFilter = Nothing
    }


 init : Flags -> ( Model, Cmd Msg )
 init flags =
    ( emptyModel
-    , Api.getFolders flags "" False GetFolderResp
+    , Cmd.batch
+        [ Api.getFolders flags "" False GetFolderResp
+        , Api.getTags flags "" GetTagResp
+        ]
    )


@@ -72,29 +84,42 @@ isValid model =
    model.abbrev /= ""


-getSource : Model -> Source
+getSource : Model -> SourceAndTags
 getSource model =
    let
-        s =
+        st =
            model.source
+
+        s =
+            st.source
+
+        tags =
+            Comp.Dropdown.getSelected model.tagModel
+
+        n =
+            { s
+                | abbrev = model.abbrev
+                , description = model.description
+                , enabled = model.enabled
+                , priority = Data.Priority.toName model.priority
+                , folder = model.folderId
+                , fileFilter = model.fileFilter
+            }
    in
-    { s
-        | abbrev = model.abbrev
-        , description = model.description
-        , enabled = model.enabled
-        , priority = Data.Priority.toName model.priority
-        , folder = model.folderId
-    }
+    { st | source = n, tags = TagList (List.length tags) tags }


 type Msg
    = SetAbbrev String
-    | SetSource Source
+    | SetSource SourceAndTags
    | SetDescr String
    | ToggleEnabled
    | PrioDropdownMsg (Comp.FixedDropdown.Msg Priority)
    | GetFolderResp (Result Http.Error FolderList)
    | FolderDropdownMsg (Comp.Dropdown.Msg IdName)
+    | GetTagResp (Result Http.Error TagList)
+    | TagDropdownMsg (Comp.Dropdown.Msg Tag)
+    | SetFileFilter String



@@ -106,29 +131,34 @@ update flags msg model =
    case msg of
        SetSource t ->
            let
-                post =
+                stpost =
                    model.source

+                post =
+                    stpost.source
+
                np =
                    { post
-                        | id = t.id
-                        , abbrev = t.abbrev
-                        , description = t.description
-                        , priority = t.priority
-                        , enabled = t.enabled
-                        , folder = t.folder
+                        | id = t.source.id
+                        , abbrev = t.source.abbrev
+                        , description = t.source.description
+                        , priority = t.source.priority
+                        , enabled = t.source.enabled
+                        , folder = t.source.folder
+                        , fileFilter = t.source.fileFilter
                    }

                newModel =
                    { model
-                        | source = np
-                        , abbrev = t.abbrev
-                        , description = t.description
+                        | source = { stpost | source = np }
+                        , abbrev = t.source.abbrev
+                        , description = t.source.description
                        , priority =
-                            Data.Priority.fromString t.priority
+                            Data.Priority.fromString t.source.priority
                                |> Maybe.withDefault Data.Priority.Low
-                        , enabled = t.enabled
-                        , folderId = t.folder
+                        , enabled = t.source.enabled
+                        , folderId = t.source.folder
+                        , fileFilter = t.source.fileFilter
                    }

                mkIdName id =
@@ -143,14 +173,21 @@ update flags msg model =
                        model.allFolders

                sel =
-                    case Maybe.map mkIdName t.folder of
+                    case Maybe.map mkIdName t.source.folder of
                        Just idref ->
                            idref

                        Nothing ->
                            []
+
+                tags =
+                    Comp.Dropdown.SetSelection t.tags.items
            in
-            update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel)) newModel
+            Util.Update.andThen1
+                [ update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel))
+                , update flags (TagDropdownMsg tags)
+                ]
+                newModel

        ToggleEnabled ->
            ( { model | enabled = not model.enabled }, Cmd.none )
@@ -159,14 +196,7 @@ update flags msg model =
            ( { model | abbrev = n }, Cmd.none )

        SetDescr d ->
-            ( { model
-                | description =
-                    if d /= "" then
-                        Just d
-
-                    else
-                        Nothing
-              }
+            ( { model | description = Util.Maybe.fromString d }
            , Cmd.none
            )

@@ -226,6 +256,31 @@ update flags msg model =
            in
            ( model_, Cmd.map FolderDropdownMsg c2 )

+        GetTagResp (Ok list) ->
+            let
+                opts =
+                    Comp.Dropdown.SetOptions list.items
+            in
+            update flags (TagDropdownMsg opts) model
+
+        GetTagResp (Err _) ->
+            ( model, Cmd.none )
+
+        TagDropdownMsg lm ->
+            let
+                ( m2, c2 ) =
+                    Comp.Dropdown.update lm model.tagModel
+
+                newModel =
+                    { model | tagModel = m2 }
+            in
+            ( newModel, Cmd.map TagDropdownMsg c2 )
+
+        SetFileFilter d ->
+            ( { model | fileFilter = Util.Maybe.fromString d }
+            , Cmd.none
+            )
+


 --- View
@@ -260,6 +315,7 @@ view flags settings model =
            , textarea
                [ onInput SetDescr
                , model.description |> Maybe.withDefault "" |> value
+                , rows 3
                ]
                []
            ]
@@ -281,12 +337,26 @@ view flags settings model =
                    (Just priorityItem)
                    model.priorityModel
                )
+            , div [ class "small-info" ]
+                [ text "The priority used by the scheduler when processing uploaded files."
+                ]
+            ]
+        , div [ class "ui dividing header" ]
+            [ text "Metadata"
+            ]
+        , div [ class "ui message" ]
+            [ text "Metadata specified here is automatically attached to each item uploaded "
+            , text "through this source, unless it is overriden in the upload request meta data. "
+            , text "Tags from the request are added to those defined here."
            ]
        , div [ class "field" ]
            [ label []
                [ text "Folder"
                ]
            , Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel)
+            , div [ class "small-info" ]
+                [ text "Choose a folder to automatically put items into."
+                ]
            , div
                [ classList
                    [ ( "ui warning message", True )
@@ -301,6 +371,33 @@ disappear then.
                      """
                ]
            ]
+        , div [ class "field" ]
+            [ label [] [ text "Tags" ]
+            , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
+            , div [ class "small-info" ]
+                [ text "Choose tags that should be applied to items."
+                ]
+            ]
+        , div
+            [ class "field"
+            ]
+            [ label [] [ text "File Filter" ]
+            , input
+                [ type_ "text"
+                , onInput SetFileFilter
+                , placeholder "File Filter"
+                , model.fileFilter
+                    |> Maybe.withDefault ""
+                    |> value
+                ]
+                []
+            , div [ class "small-info" ]
+                [ text "Specify a file glob to filter files when uploading archives (e.g. for email and zip). For example, to only extract pdf files: "
+                , code []
+                    [ text "*.pdf"
+                    ]
+                ]
+            ]
        ]


--- a/modules/webapp/src/main/elm/Comp/SourceManage.elm
+++ b/modules/webapp/src/main/elm/Comp/SourceManage.elm
@@ -8,8 +8,9 @@ module Comp.SourceManage exposing

 import Api
 import Api.Model.BasicResult exposing (BasicResult)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Comp.SourceForm
 import Comp.SourceTable exposing (SelectMode(..))
 import Comp.YesNoDimmer
@@ -31,7 +32,7 @@ type alias Model =
    , formError : Maybe String
    , loading : Bool
    , deleteConfirm : Comp.YesNoDimmer.Model
-    , sources : List Source
+    , sources : List SourceAndTags
    }


@@ -145,7 +146,7 @@ update flags msg model =
        InitNewSource ->
            let
                source =
-                    Api.Model.Source.empty
+                    Api.Model.SourceAndTags.empty

                nm =
                    { model | viewMode = Edit source, formError = Nothing }
@@ -196,7 +197,7 @@ update flags msg model =

                cmd =
                    if confirmed then
-                        Api.deleteSource flags src.id SubmitResp
+                        Api.deleteSource flags src.source.id SubmitResp

                    else
                        Cmd.none
@@ -248,22 +249,22 @@ viewTable model =
        ]


-viewLinks : Flags -> UiSettings -> Source -> Html Msg
+viewLinks : Flags -> UiSettings -> SourceAndTags -> Html Msg
 viewLinks flags _ source =
    let
        appUrl =
-            flags.config.baseUrl ++ "/app/upload/" ++ source.id
+            flags.config.baseUrl ++ "/app/upload/" ++ source.source.id

        apiUrl =
-            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.id
+            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.source.id
    in
    div
        []
        [ h3 [ class "ui dividing header" ]
            [ text "Public Uploads: "
-            , text source.abbrev
+            , text source.source.abbrev
            , div [ class "sub header" ]
-                [ text source.id
+                [ text source.source.id
                ]
            ]
        , p []
@@ -273,7 +274,7 @@ viewLinks flags _ source =
            ]
        , p []
            [ text "There have been "
-            , String.fromInt source.counter |> text
+            , String.fromInt source.source.counter |> text
            , text " items created through this source."
            ]
        , h4 [ class "ui header" ]
@@ -358,7 +359,7 @@ viewForm : Flags -> UiSettings -> Model -> List (Html Msg)
 viewForm flags settings model =
    let
        newSource =
-            model.formModel.source.id == ""
+            model.formModel.source.source.id == ""
    in
    [ if newSource then
        h3 [ class "ui top attached header" ]
@@ -367,10 +368,10 @@ viewForm flags settings model =

      else
        h3 [ class "ui top attached header" ]
-            [ text ("Edit: " ++ model.formModel.source.abbrev)
+            [ text ("Edit: " ++ model.formModel.source.source.abbrev)
            , div [ class "sub header" ]
                [ text "Id: "
-                , text model.formModel.source.id
+                , text model.formModel.source.source.id
                ]
            ]
    , Html.form [ class "ui attached segment", onSubmit Submit ]
--- a/modules/webapp/src/main/elm/Comp/SourceTable.elm
+++ b/modules/webapp/src/main/elm/Comp/SourceTable.elm
@@ -6,7 +6,7 @@ module Comp.SourceTable exposing
    , view
    )

-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Data.Flags exposing (Flags)
 import Data.Priority
 import Html exposing (..)
@@ -15,8 +15,8 @@ import Html.Events exposing (onClick)


 type SelectMode
-    = Edit Source
-    | Display Source
+    = Edit SourceAndTags
+    | Display SourceAndTags
    | None


@@ -34,8 +34,8 @@ isEdit m =


 type Msg
-    = Select Source
-    | Show Source
+    = Select SourceAndTags
+    | Show SourceAndTags


 update : Flags -> Msg -> ( Cmd Msg, SelectMode )
@@ -48,7 +48,7 @@ update _ msg =
            ( Cmd.none, Display source )


-view : List Source -> Html Msg
+view : List SourceAndTags -> Html Msg
 view sources =
    table [ class "ui table" ]
        [ thead []
@@ -66,7 +66,7 @@ view sources =
        ]


-renderSourceLine : Source -> Html Msg
+renderSourceLine : SourceAndTags -> Html Msg
 renderSourceLine source =
    tr
        []
@@ -82,10 +82,10 @@ renderSourceLine source =
            , a
                [ classList
                    [ ( "ui basic tiny primary button", True )
-                    , ( "disabled", not source.enabled )
+                    , ( "disabled", not source.source.enabled )
                    ]
                , href "#"
-                , disabled (not source.enabled)
+                , disabled (not source.source.enabled)
                , onClick (Show source)
                ]
                [ i [ class "eye icon" ] []
@@ -93,25 +93,25 @@ renderSourceLine source =
                ]
            ]
        , td [ class "collapsing" ]
-            [ text source.abbrev
+            [ text source.source.abbrev
            ]
        , td [ class "collapsing" ]
-            [ if source.enabled then
+            [ if source.source.enabled then
                i [ class "check square outline icon" ] []

              else
                i [ class "minus square outline icon" ] []
            ]
        , td [ class "collapsing" ]
-            [ source.counter |> String.fromInt |> text
+            [ source.source.counter |> String.fromInt |> text
            ]
        , td [ class "collapsing" ]
-            [ Data.Priority.fromString source.priority
+            [ Data.Priority.fromString source.source.priority
                |> Maybe.map Data.Priority.toName
-                |> Maybe.withDefault source.priority
+                |> Maybe.withDefault source.source.priority
                |> text
            ]
        , td []
-            [ text source.id
+            [ text source.source.id
            ]
        ]