Amend source form with tags and file-filter

Allow to define tags and a file filter per source.

modules/backend/src/main/scala/docspell/backend/ops/OSource.scala

@@ -4,44 +4,50 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._
 
 import docspell.common.{AccountId, Ident}
+import docspell.store.UpdateResult
 import docspell.store.records.RSource
+import docspell.store.records.SourceData
 import docspell.store.{AddResult, Store}
 
 trait OSource[F[_]] {
 
-  def findAll(account: AccountId): F[Vector[RSource]]
+  def findAll(account: AccountId): F[Vector[SourceData]]
 
-  def add(s: RSource): F[AddResult]
+  def add(s: RSource, tags: List[String]): F[AddResult]
 
-  def update(s: RSource): F[AddResult]
+  def update(s: RSource, tags: List[String]): F[AddResult]
 
-  def delete(id: Ident, collective: Ident): F[AddResult]
+  def delete(id: Ident, collective: Ident): F[UpdateResult]
 }
 
 object OSource {
 
   def apply[F[_]: Effect](store: Store[F]): Resource[F, OSource[F]] =
     Resource.pure[F, OSource[F]](new OSource[F] {
-      def findAll(account: AccountId): F[Vector[RSource]] =
+      def findAll(account: AccountId): F[Vector[SourceData]] =
-        store.transact(RSource.findAll(account.collective, _.abbrev))
+        store
+          .transact(SourceData.findAll(account.collective, _.abbrev))
+          .compile
+          .to(Vector)
 
-      def add(s: RSource): F[AddResult] = {
+      def add(s: RSource, tags: List[String]): F[AddResult] = {
-        def insert = RSource.insert(s)
+        def insert = SourceData.insert(s, tags)
         def exists = RSource.existsByAbbrev(s.cid, s.abbrev)
 
         val msg = s"A source with abbrev '${s.abbrev}' already exists"
         store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
       }
 
-      def update(s: RSource): F[AddResult] = {
+      def update(s: RSource, tags: List[String]): F[AddResult] = {
-        def insert = RSource.updateNoCounter(s)
+        def insert = SourceData.update(s, tags)
         def exists = RSource.existsByAbbrev(s.cid, s.abbrev)
 
         val msg = s"A source with abbrev '${s.abbrev}' already exists"
         store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
       }
 
-      def delete(id: Ident, collective: Ident): F[AddResult] =
+      def delete(id: Ident, collective: Ident): F[UpdateResult] =
-        store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
+        UpdateResult.fromUpdate(store.transact(SourceData.delete(id, collective)))
+
     })
 }

modules/backend/src/main/scala/docspell/backend/ops/OTag.scala

@@ -4,6 +4,7 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._
 
 import docspell.common.{AccountId, Ident}
+import docspell.store.records.RTagSource
 import docspell.store.records.{RTag, RTagItem}
 import docspell.store.{AddResult, Store}
 
@@ -49,8 +50,9 @@ object OTag {
         val io = for {
           optTag <- RTag.findByIdAndCollective(id, collective)
           n0     <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
-          n1     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
+          n1     <- optTag.traverse(t => RTagSource.deleteTag(t.tagId))
-        } yield n0.getOrElse(0) + n1.getOrElse(0)
+          n2     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
+        } yield (n0 |+| n1 |+| n2).getOrElse(0)
         store.transact(io).attempt.map(AddResult.fromUpdate)
       }
 

modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala

@@ -25,6 +25,11 @@ trait OUpload[F[_]] {
       itemId: Option[Ident]
   ): F[OUpload.UploadResult]
 
+  /** Submit files via a given source identifier. The source is looked
+    * up to identify the collective the files belong to. Metadata
+    * defined in the source is used as a fallback to those specified
+    * here (in UploadData).
+    */
   def submit(
       data: OUpload.UploadData[F],
       sourceId: Ident,
@@ -153,15 +158,19 @@ object OUpload {
           itemId: Option[Ident]
       ): F[OUpload.UploadResult] =
         (for {
-          src <- OptionT(store.transact(RSource.findEnabled(sourceId)))
+          src <- OptionT(store.transact(SourceData.findEnabled(sourceId)))
           updata = data.copy(
             meta = data.meta.copy(
-              sourceAbbrev = src.abbrev,
+              sourceAbbrev = src.source.abbrev,
-              folderId = data.meta.folderId.orElse(src.folderId)
+              folderId = data.meta.folderId.orElse(src.source.folderId),
+              fileFilter =
+                if (data.meta.fileFilter == Glob.all) src.source.fileFilterOrAll
+                else data.meta.fileFilter,
+              tags = (data.meta.tags ++ src.tags.map(_.tagId.id)).distinct
             ),
-            priority = src.priority
+            priority = src.source.priority
           )
-          accId = AccountId(src.cid, src.sid)
+          accId = AccountId(src.source.cid, src.source.sid)
           result <- OptionT.liftF(submit(updata, accId, notifyJoex, itemId))
         } yield result).getOrElse(UploadResult.noSource)
 

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -1211,7 +1211,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
       responses:
         200:
           description: Ok
@@ -1231,7 +1231,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
       responses:
         200:
           description: Ok
@@ -4366,7 +4366,7 @@ components:
         items:
           type: array
           items:
-            $ref: "#/components/schemas/Source"
+            $ref: "#/components/schemas/SourceAndTags"
     Source:
       description: |
         Data about a Source. A source defines the endpoint where
@@ -4400,10 +4400,38 @@ components:
         folder:
           type: string
           format: ident
+        fileFilter:
+          type: string
+          format: glob
         created:
           description: DateTime
           type: integer
           format: date-time
+    SourceTagIn:
+      description: |
+        A source and optional tags (ids or names) for updating/adding.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          type: array
+          items:
+            type: string
+    SourceAndTags:
+      description: |
+        A source and optional tags.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          $ref: "#/components/schema/TagList"
+
     EquipmentList:
       description: |
         A list of equipments.

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -524,21 +524,36 @@ trait Conversions {
 
   // sources
 
-  def mkSource(s: RSource): Source =
+  def mkSource(s: SourceData): SourceAndTags =
-    Source(
+    SourceAndTags(
-      s.sid,
+      Source(
-      s.abbrev,
+        s.source.sid,
-      s.description,
+        s.source.abbrev,
-      s.counter,
+        s.source.description,
-      s.enabled,
+        s.source.counter,
-      s.priority,
+        s.source.enabled,
-      s.folderId,
+        s.source.priority,
-      s.created
+        s.source.folderId,
+        s.source.fileFilter,
+        s.source.created
+      ),
+      TagList(s.tags.length, s.tags.map(mkTag).toList)
     )
 
   def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
     timeId.map({ case (id, now) =>
-      RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder)
+      RSource(
+        id,
+        cid,
+        s.abbrev,
+        s.description,
+        0,
+        s.enabled,
+        s.priority,
+        now,
+        s.folder,
+        s.fileFilter
+      )
     })
 
   def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
@@ -551,7 +566,8 @@ trait Conversions {
       s.enabled,
       s.priority,
       s.created,
-      s.folder
+      s.folder,
+      s.fileFilter
     )
 
   // equipment

modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala

@@ -30,17 +30,17 @@ object SourceRoutes {
 
       case req @ POST -> Root =>
         for {
-          data  <- req.as[Source]
+          data  <- req.as[SourceTagIn]
-          src   <- newSource(data, user.account.collective)
+          src   <- newSource(data.source, user.account.collective)
-          added <- backend.source.add(src)
+          added <- backend.source.add(src, data.tags)
           resp  <- Ok(basicResult(added, "Source added."))
         } yield resp
 
       case req @ PUT -> Root =>
         for {
-          data <- req.as[Source]
+          data <- req.as[SourceTagIn]
-          src = changeSource(data, user.account.collective)
+          src = changeSource(data.source, user.account.collective)
-          updated <- backend.source.update(src)
+          updated <- backend.source.update(src, data.tags)
           resp    <- Ok(basicResult(updated, "Source updated."))
         } yield resp
 

modules/store/src/main/resources/db/migration/h2/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/resources/db/migration/mariadb/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE `source`
+ADD COLUMN `file_filter` varchar(254) NULL;
+
+CREATE TABLE `tagsource` (
+  `id` varchar(254) not null primary key,
+  `source_id` varchar(254) not null,
+  `tag_id` varchar(254) not null,
+  unique (`source_id`, `tag_id`),
+  foreign key (`source_id`) references `source`(`sid`),
+  foreign key (`tag_id`) references `tag`(`tid`)
+);

modules/store/src/main/resources/db/migration/postgresql/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala

@@ -91,6 +91,9 @@ trait DoobieMeta extends EmilDoobieMeta {
 
   implicit val metaCalEvent: Meta[CalEvent] =
     Meta[String].timap(CalEvent.unsafe)(_.asString)
+
+  implicit val metaGlob: Meta[Glob] =
+    Meta[String].timap(Glob.apply)(_.asString)
 }
 
 object DoobieMeta extends DoobieMeta {

modules/store/src/main/scala/docspell/store/records/RSource.scala

@@ -16,8 +16,13 @@ case class RSource(
     enabled: Boolean,
     priority: Priority,
     created: Timestamp,
-    folderId: Option[Ident]
+    folderId: Option[Ident],
-) {}
+    fileFilter: Option[Glob]
+) {
+
+  def fileFilterOrAll: Glob =
+    fileFilter.getOrElse(Glob.all)
+}
 
 object RSource {
 
@@ -34,9 +39,21 @@ object RSource {
     val priority    = Column("priority")
     val created     = Column("created")
     val folder      = Column("folder_id")
+    val fileFilter  = Column("file_filter")
 
     val all =
-      List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
+      List(
+        sid,
+        cid,
+        abbrev,
+        description,
+        counter,
+        enabled,
+        priority,
+        created,
+        folder,
+        fileFilter
+      )
   }
 
   import Columns._
@@ -45,7 +62,7 @@ object RSource {
     val sql = insertRow(
       table,
       all,
-      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
+      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}"
     )
     sql.update.run
   }
@@ -60,7 +77,8 @@ object RSource {
         description.setTo(v.description),
         enabled.setTo(v.enabled),
         priority.setTo(v.priority),
-        folder.setTo(v.folderId)
+        folder.setTo(v.folderId),
+        fileFilter.setTo(v.fileFilter)
       )
     )
     sql.update.run
@@ -83,10 +101,11 @@ object RSource {
     sql.query[Int].unique.map(_ > 0)
   }
 
-  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] = {
+  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] =
-    val sql = selectSimple(all, table, and(sid.is(id), enabled.is(true)))
+    findEnabledSql(id).query[RSource].option
-    sql.query[RSource].option
+
-  }
+  private[records] def findEnabledSql(id: Ident): Fragment =
+    selectSimple(all, table, and(sid.is(id), enabled.is(true)))
 
   def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] =
     selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option
@@ -94,10 +113,11 @@ object RSource {
   def findAll(
       coll: Ident,
       order: Columns.type => Column
-  ): ConnectionIO[Vector[RSource]] = {
+  ): ConnectionIO[Vector[RSource]] =
-    val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
+    findAllSql(coll, order).query[RSource].to[Vector]
-    sql.query[RSource].to[Vector]
+
-  }
+  private[records] def findAllSql(coll: Ident, order: Columns.type => Column): Fragment =
+    selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
 
   def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
     deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run

modules/store/src/main/scala/docspell/store/records/RTag.scala

@@ -104,6 +104,18 @@ object RTag {
     ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
   }
 
+  def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = {
+    val rcol = all.map(_.prefix("t"))
+    (selectSimple(
+      rcol,
+      table ++ fr"t," ++ RTagSource.table ++ fr"s",
+      and(
+        RTagSource.Columns.sourceId.prefix("s").is(source),
+        RTagSource.Columns.tagId.prefix("s").is(tid.prefix("t"))
+      )
+    ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
+  }
+
   def findAllByNameOrId(
       nameOrIds: List[String],
       coll: Ident

modules/store/src/main/scala/docspell/store/records/RTagSource.scala

@@ -0,0 +1,56 @@
+package docspell.store.records
+
+import cats.effect.Sync
+import cats.implicits._
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+case class RTagSource(id: Ident, sourceId: Ident, tagId: Ident) {}
+
+object RTagSource {
+
+  val table = fr"tagsource"
+
+  object Columns {
+    val id       = Column("id")
+    val sourceId = Column("source_id")
+    val tagId    = Column("tag_id")
+    val all      = List(id, sourceId, tagId)
+  }
+  import Columns._
+
+  def createNew[F[_]: Sync](source: Ident, tag: Ident): F[RTagSource] =
+    Ident.randomId[F].map(id => RTagSource(id, source, tag))
+
+  def insert(v: RTagSource): ConnectionIO[Int] =
+    insertRow(table, all, fr"${v.id},${v.sourceId},${v.tagId}").update.run
+
+  def deleteSourceTags(source: Ident): ConnectionIO[Int] =
+    deleteFrom(table, sourceId.is(source)).update.run
+
+  def deleteTag(tid: Ident): ConnectionIO[Int] =
+    deleteFrom(table, tagId.is(tid)).update.run
+
+  def findBySource(source: Ident): ConnectionIO[Vector[RTagSource]] =
+    selectSimple(all, table, sourceId.is(source)).query[RTagSource].to[Vector]
+
+  def setAllTags(source: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
+    if (tags.isEmpty) 0.pure[ConnectionIO]
+    else
+      for {
+        entities <- tags.toList.traverse(tagId =>
+          Ident.randomId[ConnectionIO].map(id => RTagSource(id, source, tagId))
+        )
+        n <- insertRows(
+          table,
+          all,
+          entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}")
+        ).update.run
+      } yield n
+
+}

modules/store/src/main/scala/docspell/store/records/SourceData.scala

@@ -0,0 +1,87 @@
+package docspell.store.records
+
+import cats.effect.concurrent.Ref
+import cats.implicits._
+import fs2.Stream
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+/** Combines a source record (RSource) and a list of associated tags.
+  */
+case class SourceData(source: RSource, tags: Vector[RTag])
+
+object SourceData {
+
+  def fromSource(s: RSource): SourceData =
+    SourceData(s, Vector.empty)
+
+  def findAll(
+      coll: Ident,
+      order: RSource.Columns.type => Column
+  ): Stream[ConnectionIO, SourceData] =
+    findAllWithTags(RSource.findAllSql(coll, order).query[RSource].stream)
+
+  private def findAllWithTags(
+      select: Stream[ConnectionIO, RSource]
+  ): Stream[ConnectionIO, SourceData] = {
+    def findTag(
+        cache: Ref[ConnectionIO, Map[Ident, RTag]],
+        tagSource: RTagSource
+    ): ConnectionIO[Option[RTag]] =
+      for {
+        cc <- cache.get
+        fromCache = cc.get(tagSource.tagId)
+        orFromDB <-
+          if (fromCache.isDefined) fromCache.pure[ConnectionIO]
+          else RTag.findById(tagSource.tagId)
+        _ <-
+          if (fromCache.isDefined) ().pure[ConnectionIO]
+          else
+            orFromDB match {
+              case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
+              case None    => ().pure[ConnectionIO]
+            }
+      } yield orFromDB
+
+    for {
+      resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
+      source       <- select
+      tagSources   <- Stream.eval(RTagSource.findBySource(source.sid))
+      tags         <- Stream.eval(tagSources.traverse(ti => findTag(resolvedTags, ti)))
+    } yield SourceData(source, tags.flatten)
+  }
+
+  def findEnabled(id: Ident): ConnectionIO[Option[SourceData]] =
+    findAllWithTags(RSource.findEnabledSql(id).query[RSource].stream).head.compile.last
+
+  def insert(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.insert(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def update(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.updateNoCounter(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      _    <- RTagSource.deleteSourceTags(data.sid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def delete(source: Ident, coll: Ident): ConnectionIO[Int] =
+    for {
+      n0 <- RTagSource.deleteSourceTags(source)
+      n1 <- RSource.delete(source, coll)
+    } yield n0 + n1
+
+}

modules/webapp/src/main/elm/Api.elm

@@ -175,8 +175,9 @@ import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
 import Api.Model.ScanMailboxSettingsList exposing (ScanMailboxSettingsList)
 import Api.Model.SentMails exposing (SentMails)
 import Api.Model.SimpleMail exposing (SimpleMail)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Api.Model.StringList exposing (StringList)
 import Api.Model.Tag exposing (Tag)
 import Api.Model.TagCloud exposing (TagCloud)
@@ -1144,17 +1145,22 @@ getSources flags receive =
         }
 
 
-postSource : Flags -> Source -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+postSource : Flags -> SourceAndTags -> (Result Http.Error BasicResult -> msg) -> Cmd msg
 postSource flags source receive =
     let
+        st =
+            { source = source.source
+            , tags = List.map .id source.tags.items
+            }
+
         params =
             { url = flags.config.baseUrl ++ "/api/v1/sec/source"
             , account = getAccount flags
-            , body = Http.jsonBody (Api.Model.Source.encode source)
+            , body = Http.jsonBody (Api.Model.SourceTagIn.encode st)
             , expect = Http.expectJson receive Api.Model.BasicResult.decoder
             }
     in
-    if source.id == "" then
+    if source.source.id == "" then
         Http2.authPost params
 
     else

modules/webapp/src/main/elm/Comp/SourceForm.elm

@@ -12,7 +12,9 @@ import Api
 import Api.Model.FolderItem exposing (FolderItem)
 import Api.Model.FolderList exposing (FolderList)
 import Api.Model.IdName exposing (IdName)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
+import Api.Model.Tag exposing (Tag)
+import Api.Model.TagList exposing (TagList)
 import Comp.Dropdown exposing (isDropdownChangeMsg)
 import Comp.FixedDropdown
 import Data.Flags exposing (Flags)
@@ -24,10 +26,13 @@ import Html.Events exposing (onCheck, onInput)
 import Http
 import Markdown
 import Util.Folder exposing (mkFolderOption)
+import Util.Maybe
+import Util.Tag
+import Util.Update
 
 
 type alias Model =
-    { source : Source
+    { source : SourceAndTags
     , abbrev : String
     , description : Maybe String
     , priorityModel : Comp.FixedDropdown.Model Priority
@@ -36,12 +41,14 @@ type alias Model =
     , folderModel : Comp.Dropdown.Model IdName
     , allFolders : List FolderItem
     , folderId : Maybe String
+    , tagModel : Comp.Dropdown.Model Tag
+    , fileFilter : Maybe String
     }
 
 
 emptyModel : Model
 emptyModel =
-    { source = Api.Model.Source.empty
+    { source = Api.Model.SourceAndTags.empty
     , abbrev = ""
     , description = Nothing
     , priorityModel =
@@ -57,13 +64,18 @@ emptyModel =
             }
     , allFolders = []
     , folderId = Nothing
+    , tagModel = Util.Tag.makeDropdownModel
+    , fileFilter = Nothing
     }
 
 
 init : Flags -> ( Model, Cmd Msg )
 init flags =
     ( emptyModel
-    , Api.getFolders flags "" False GetFolderResp
+    , Cmd.batch
+        [ Api.getFolders flags "" False GetFolderResp
+        , Api.getTags flags "" GetTagResp
+        ]
     )
 
 
@@ -72,29 +84,42 @@ isValid model =
     model.abbrev /= ""
 
 
-getSource : Model -> Source
+getSource : Model -> SourceAndTags
 getSource model =
     let
-        s =
+        st =
             model.source
+
+        s =
+            st.source
+
+        tags =
+            Comp.Dropdown.getSelected model.tagModel
+
+        n =
+            { s
+                | abbrev = model.abbrev
+                , description = model.description
+                , enabled = model.enabled
+                , priority = Data.Priority.toName model.priority
+                , folder = model.folderId
+                , fileFilter = model.fileFilter
+            }
     in
-    { s
+    { st | source = n, tags = TagList (List.length tags) tags }
-        | abbrev = model.abbrev
-        , description = model.description
-        , enabled = model.enabled
-        , priority = Data.Priority.toName model.priority
-        , folder = model.folderId
-    }
 
 
 type Msg
     = SetAbbrev String
-    | SetSource Source
+    | SetSource SourceAndTags
     | SetDescr String
     | ToggleEnabled
     | PrioDropdownMsg (Comp.FixedDropdown.Msg Priority)
     | GetFolderResp (Result Http.Error FolderList)
     | FolderDropdownMsg (Comp.Dropdown.Msg IdName)
+    | GetTagResp (Result Http.Error TagList)
+    | TagDropdownMsg (Comp.Dropdown.Msg Tag)
+    | SetFileFilter String
 
 
 
@@ -106,29 +131,34 @@ update flags msg model =
     case msg of
         SetSource t ->
             let
-                post =
+                stpost =
                     model.source
 
+                post =
+                    stpost.source
+
                 np =
                     { post
-                        | id = t.id
+                        | id = t.source.id
-                        , abbrev = t.abbrev
+                        , abbrev = t.source.abbrev
-                        , description = t.description
+                        , description = t.source.description
-                        , priority = t.priority
+                        , priority = t.source.priority
-                        , enabled = t.enabled
+                        , enabled = t.source.enabled
-                        , folder = t.folder
+                        , folder = t.source.folder
+                        , fileFilter = t.source.fileFilter
                     }
 
                 newModel =
                     { model
-                        | source = np
+                        | source = { stpost | source = np }
-                        , abbrev = t.abbrev
+                        , abbrev = t.source.abbrev
-                        , description = t.description
+                        , description = t.source.description
                         , priority =
-                            Data.Priority.fromString t.priority
+                            Data.Priority.fromString t.source.priority
                                 |> Maybe.withDefault Data.Priority.Low
-                        , enabled = t.enabled
+                        , enabled = t.source.enabled
-                        , folderId = t.folder
+                        , folderId = t.source.folder
+                        , fileFilter = t.source.fileFilter
                     }
 
                 mkIdName id =
@@ -143,14 +173,21 @@ update flags msg model =
                         model.allFolders
 
                 sel =
-                    case Maybe.map mkIdName t.folder of
+                    case Maybe.map mkIdName t.source.folder of
                         Just idref ->
                             idref
 
                         Nothing ->
                             []
+
+                tags =
+                    Comp.Dropdown.SetSelection t.tags.items
             in
-            update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel)) newModel
+            Util.Update.andThen1
+                [ update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel))
+                , update flags (TagDropdownMsg tags)
+                ]
+                newModel
 
         ToggleEnabled ->
             ( { model | enabled = not model.enabled }, Cmd.none )
@@ -159,14 +196,7 @@ update flags msg model =
             ( { model | abbrev = n }, Cmd.none )
 
         SetDescr d ->
-            ( { model
+            ( { model | description = Util.Maybe.fromString d }
-                | description =
-                    if d /= "" then
-                        Just d
-
-                    else
-                        Nothing
-              }
             , Cmd.none
             )
 
@@ -226,6 +256,31 @@ update flags msg model =
             in
             ( model_, Cmd.map FolderDropdownMsg c2 )
 
+        GetTagResp (Ok list) ->
+            let
+                opts =
+                    Comp.Dropdown.SetOptions list.items
+            in
+            update flags (TagDropdownMsg opts) model
+
+        GetTagResp (Err _) ->
+            ( model, Cmd.none )
+
+        TagDropdownMsg lm ->
+            let
+                ( m2, c2 ) =
+                    Comp.Dropdown.update lm model.tagModel
+
+                newModel =
+                    { model | tagModel = m2 }
+            in
+            ( newModel, Cmd.map TagDropdownMsg c2 )
+
+        SetFileFilter d ->
+            ( { model | fileFilter = Util.Maybe.fromString d }
+            , Cmd.none
+            )
+
 
 
 --- View
@@ -260,6 +315,7 @@ view flags settings model =
             , textarea
                 [ onInput SetDescr
                 , model.description |> Maybe.withDefault "" |> value
+                , rows 3
                 ]
                 []
             ]
@@ -281,12 +337,26 @@ view flags settings model =
                     (Just priorityItem)
                     model.priorityModel
                 )
+            , div [ class "small-info" ]
+                [ text "The priority used by the scheduler when processing uploaded files."
+                ]
+            ]
+        , div [ class "ui dividing header" ]
+            [ text "Metadata"
+            ]
+        , div [ class "ui message" ]
+            [ text "Metadata specified here is automatically attached to each item uploaded "
+            , text "through this source, unless it is overriden in the upload request meta data. "
+            , text "Tags from the request are added to those defined here."
             ]
         , div [ class "field" ]
             [ label []
                 [ text "Folder"
                 ]
             , Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel)
+            , div [ class "small-info" ]
+                [ text "Choose a folder to automatically put items into."
+                ]
             , div
                 [ classList
                     [ ( "ui warning message", True )
@@ -301,6 +371,33 @@ disappear then.
                       """
                 ]
             ]
+        , div [ class "field" ]
+            [ label [] [ text "Tags" ]
+            , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
+            , div [ class "small-info" ]
+                [ text "Choose tags that should be applied to items."
+                ]
+            ]
+        , div
+            [ class "field"
+            ]
+            [ label [] [ text "File Filter" ]
+            , input
+                [ type_ "text"
+                , onInput SetFileFilter
+                , placeholder "File Filter"
+                , model.fileFilter
+                    |> Maybe.withDefault ""
+                    |> value
+                ]
+                []
+            , div [ class "small-info" ]
+                [ text "Specify a file glob to filter files when uploading archives (e.g. for email and zip). For example, to only extract pdf files: "
+                , code []
+                    [ text "*.pdf"
+                    ]
+                ]
+            ]
         ]
 
 

modules/webapp/src/main/elm/Comp/SourceManage.elm

@@ -8,8 +8,9 @@ module Comp.SourceManage exposing
 
 import Api
 import Api.Model.BasicResult exposing (BasicResult)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Comp.SourceForm
 import Comp.SourceTable exposing (SelectMode(..))
 import Comp.YesNoDimmer
@@ -31,7 +32,7 @@ type alias Model =
     , formError : Maybe String
     , loading : Bool
     , deleteConfirm : Comp.YesNoDimmer.Model
-    , sources : List Source
+    , sources : List SourceAndTags
     }
 
 
@@ -145,7 +146,7 @@ update flags msg model =
         InitNewSource ->
             let
                 source =
-                    Api.Model.Source.empty
+                    Api.Model.SourceAndTags.empty
 
                 nm =
                     { model | viewMode = Edit source, formError = Nothing }
@@ -196,7 +197,7 @@ update flags msg model =
 
                 cmd =
                     if confirmed then
-                        Api.deleteSource flags src.id SubmitResp
+                        Api.deleteSource flags src.source.id SubmitResp
 
                     else
                         Cmd.none
@@ -248,22 +249,22 @@ viewTable model =
         ]
 
 
-viewLinks : Flags -> UiSettings -> Source -> Html Msg
+viewLinks : Flags -> UiSettings -> SourceAndTags -> Html Msg
 viewLinks flags _ source =
     let
         appUrl =
-            flags.config.baseUrl ++ "/app/upload/" ++ source.id
+            flags.config.baseUrl ++ "/app/upload/" ++ source.source.id
 
         apiUrl =
-            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.id
+            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.source.id
     in
     div
         []
         [ h3 [ class "ui dividing header" ]
             [ text "Public Uploads: "
-            , text source.abbrev
+            , text source.source.abbrev
             , div [ class "sub header" ]
-                [ text source.id
+                [ text source.source.id
                 ]
             ]
         , p []
@@ -273,7 +274,7 @@ viewLinks flags _ source =
             ]
         , p []
             [ text "There have been "
-            , String.fromInt source.counter |> text
+            , String.fromInt source.source.counter |> text
             , text " items created through this source."
             ]
         , h4 [ class "ui header" ]
@@ -358,7 +359,7 @@ viewForm : Flags -> UiSettings -> Model -> List (Html Msg)
 viewForm flags settings model =
     let
         newSource =
-            model.formModel.source.id == ""
+            model.formModel.source.source.id == ""
     in
     [ if newSource then
         h3 [ class "ui top attached header" ]
@@ -367,10 +368,10 @@ viewForm flags settings model =
 
       else
         h3 [ class "ui top attached header" ]
-            [ text ("Edit: " ++ model.formModel.source.abbrev)
+            [ text ("Edit: " ++ model.formModel.source.source.abbrev)
             , div [ class "sub header" ]
                 [ text "Id: "
-                , text model.formModel.source.id
+                , text model.formModel.source.source.id
                 ]
             ]
     , Html.form [ class "ui attached segment", onSubmit Submit ]

modules/webapp/src/main/elm/Comp/SourceTable.elm

@@ -6,7 +6,7 @@ module Comp.SourceTable exposing
     , view
     )
 
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Data.Flags exposing (Flags)
 import Data.Priority
 import Html exposing (..)
@@ -15,8 +15,8 @@ import Html.Events exposing (onClick)
 
 
 type SelectMode
-    = Edit Source
+    = Edit SourceAndTags
-    | Display Source
+    | Display SourceAndTags
     | None
 
 
@@ -34,8 +34,8 @@ isEdit m =
 
 
 type Msg
-    = Select Source
+    = Select SourceAndTags
-    | Show Source
+    | Show SourceAndTags
 
 
 update : Flags -> Msg -> ( Cmd Msg, SelectMode )
@@ -48,7 +48,7 @@ update _ msg =
             ( Cmd.none, Display source )
 
 
-view : List Source -> Html Msg
+view : List SourceAndTags -> Html Msg
 view sources =
     table [ class "ui table" ]
         [ thead []
@@ -66,7 +66,7 @@ view sources =
         ]
 
 
-renderSourceLine : Source -> Html Msg
+renderSourceLine : SourceAndTags -> Html Msg
 renderSourceLine source =
     tr
         []
@@ -82,10 +82,10 @@ renderSourceLine source =
             , a
                 [ classList
                     [ ( "ui basic tiny primary button", True )
-                    , ( "disabled", not source.enabled )
+                    , ( "disabled", not source.source.enabled )
                     ]
                 , href "#"
-                , disabled (not source.enabled)
+                , disabled (not source.source.enabled)
                 , onClick (Show source)
                 ]
                 [ i [ class "eye icon" ] []
@@ -93,25 +93,25 @@ renderSourceLine source =
                 ]
             ]
         , td [ class "collapsing" ]
-            [ text source.abbrev
+            [ text source.source.abbrev
             ]
         , td [ class "collapsing" ]
-            [ if source.enabled then
+            [ if source.source.enabled then
                 i [ class "check square outline icon" ] []
 
               else
                 i [ class "minus square outline icon" ] []
             ]
         , td [ class "collapsing" ]
-            [ source.counter |> String.fromInt |> text
+            [ source.source.counter |> String.fromInt |> text
             ]
         , td [ class "collapsing" ]
-            [ Data.Priority.fromString source.priority
+            [ Data.Priority.fromString source.source.priority
                 |> Maybe.map Data.Priority.toName
-                |> Maybe.withDefault source.priority
+                |> Maybe.withDefault source.source.priority
                 |> text
             ]
         , td []
-            [ text source.id
+            [ text source.source.id
             ]
         ]