Merge pull request #437 from eikek/upload-improvements

Upload improvements

build.sbt

@@ -124,9 +124,8 @@ val buildInfoSettings = Seq(
 val openapiScalaSettings = Seq(
   openapiScalaConfig := ScalaConfig()
     .withJson(ScalaJson.circeSemiauto)
-    .addMapping(CustomMapping.forType({
+    .addMapping(CustomMapping.forType({ case TypeDef("LocalDateTime", _) =>
-      case TypeDef("LocalDateTime", _) =>
+      TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
-        TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
     }))
     .addMapping(CustomMapping.forFormatType({
       case "ident" =>
@@ -182,6 +181,8 @@ val openapiScalaSettings = Seq(
               )
             )
           )
+      case "glob" =>
+        field => field.copy(typeDef = TypeDef("Glob", Imports("docspell.common.Glob")))
     }))
 )
 
@@ -595,11 +596,10 @@ def copyWebjarResources(
   src.flatMap { dir =>
     if (dir.isDirectory) {
       val files = (dir ** "*").filter(_.isFile).get.pair(Path.relativeTo(dir))
-      files.flatMap {
+      files.flatMap { case (f, name) =>
-        case (f, name) =>
+        val target = targetDir / name
-          val target = targetDir / name
+        IO.createDirectories(Seq(target.getParentFile))
-          IO.createDirectories(Seq(target.getParentFile))
+        copyWithGZ(f, target)
-          copyWithGZ(f, target)
       }
     } else {
       val target = targetDir / dir.name
@@ -633,11 +633,13 @@ def compileElm(
 }
 
 def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = {
-  val target = out / "Webjars.scala"
+  val target   = out / "Webjars.scala"
   val badChars = "-.".toSet
   val fields = wj
     .map(m =>
-      s"""val ${m.name.toLowerCase.filter(c => !badChars.contains(c))} = "/${m.name}/${m.revision}" """
+      s"""val ${m.name.toLowerCase.filter(c =>
+        !badChars.contains(c)
+      )} = "/${m.name}/${m.revision}" """
     )
     .mkString("\n\n")
   val content = s"""package docspell.restserver.webapp

modules/backend/src/main/scala/docspell/backend/ops/OSource.scala

@@ -4,44 +4,50 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._
 
 import docspell.common.{AccountId, Ident}
+import docspell.store.UpdateResult
 import docspell.store.records.RSource
+import docspell.store.records.SourceData
 import docspell.store.{AddResult, Store}
 
 trait OSource[F[_]] {
 
-  def findAll(account: AccountId): F[Vector[RSource]]
+  def findAll(account: AccountId): F[Vector[SourceData]]
 
-  def add(s: RSource): F[AddResult]
+  def add(s: RSource, tags: List[String]): F[AddResult]
 
-  def update(s: RSource): F[AddResult]
+  def update(s: RSource, tags: List[String]): F[AddResult]
 
-  def delete(id: Ident, collective: Ident): F[AddResult]
+  def delete(id: Ident, collective: Ident): F[UpdateResult]
 }
 
 object OSource {
 
   def apply[F[_]: Effect](store: Store[F]): Resource[F, OSource[F]] =
     Resource.pure[F, OSource[F]](new OSource[F] {
-      def findAll(account: AccountId): F[Vector[RSource]] =
+      def findAll(account: AccountId): F[Vector[SourceData]] =
-        store.transact(RSource.findAll(account.collective, _.abbrev))
+        store
+          .transact(SourceData.findAll(account.collective, _.abbrev))
+          .compile
+          .to(Vector)
 
-      def add(s: RSource): F[AddResult] = {
+      def add(s: RSource, tags: List[String]): F[AddResult] = {
-        def insert = RSource.insert(s)
+        def insert = SourceData.insert(s, tags)
         def exists = RSource.existsByAbbrev(s.cid, s.abbrev)
 
         val msg = s"A source with abbrev '${s.abbrev}' already exists"
         store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
       }
 
-      def update(s: RSource): F[AddResult] = {
+      def update(s: RSource, tags: List[String]): F[AddResult] = {
-        def insert = RSource.updateNoCounter(s)
+        def insert = SourceData.update(s, tags)
         def exists = RSource.existsByAbbrev(s.cid, s.abbrev)
 
         val msg = s"A source with abbrev '${s.abbrev}' already exists"
         store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
       }
 
-      def delete(id: Ident, collective: Ident): F[AddResult] =
+      def delete(id: Ident, collective: Ident): F[UpdateResult] =
-        store.transact(RSource.delete(id, collective)).attempt.map(AddResult.fromUpdate)
+        UpdateResult.fromUpdate(store.transact(SourceData.delete(id, collective)))
+
     })
 }

modules/backend/src/main/scala/docspell/backend/ops/OTag.scala

@@ -4,6 +4,7 @@ import cats.effect.{Effect, Resource}
 import cats.implicits._
 
 import docspell.common.{AccountId, Ident}
+import docspell.store.records.RTagSource
 import docspell.store.records.{RTag, RTagItem}
 import docspell.store.{AddResult, Store}
 
@@ -49,8 +50,9 @@ object OTag {
         val io = for {
           optTag <- RTag.findByIdAndCollective(id, collective)
           n0     <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
-          n1     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
+          n1     <- optTag.traverse(t => RTagSource.deleteTag(t.tagId))
-        } yield n0.getOrElse(0) + n1.getOrElse(0)
+          n2     <- optTag.traverse(t => RTag.delete(t.tagId, collective))
+        } yield (n0 |+| n1 |+| n2).getOrElse(0)
         store.transact(io).attempt.map(AddResult.fromUpdate)
       }
 

modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala

@@ -25,6 +25,11 @@ trait OUpload[F[_]] {
       itemId: Option[Ident]
   ): F[OUpload.UploadResult]
 
+  /** Submit files via a given source identifier. The source is looked
+    * up to identify the collective the files belong to. Metadata
+    * defined in the source is used as a fallback to those specified
+    * here (in UploadData).
+    */
   def submit(
       data: OUpload.UploadData[F],
       sourceId: Ident,
@@ -60,7 +65,9 @@ object OUpload {
       sourceAbbrev: String,
       folderId: Option[Ident],
       validFileTypes: Seq[MimeType],
-      skipDuplicates: Boolean
+      skipDuplicates: Boolean,
+      fileFilter: Glob,
+      tags: List[String]
   )
 
   case class UploadData[F[_]](
@@ -127,7 +134,9 @@ object OUpload {
             data.meta.sourceAbbrev,
             data.meta.folderId,
             data.meta.validFileTypes,
-            data.meta.skipDuplicates
+            data.meta.skipDuplicates,
+            data.meta.fileFilter.some,
+            data.meta.tags.some
           )
           args =
             if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
@@ -149,15 +158,19 @@ object OUpload {
           itemId: Option[Ident]
       ): F[OUpload.UploadResult] =
         (for {
-          src <- OptionT(store.transact(RSource.findEnabled(sourceId)))
+          src <- OptionT(store.transact(SourceData.findEnabled(sourceId)))
           updata = data.copy(
             meta = data.meta.copy(
-              sourceAbbrev = src.abbrev,
+              sourceAbbrev = src.source.abbrev,
-              folderId = data.meta.folderId.orElse(src.folderId)
+              folderId = data.meta.folderId.orElse(src.source.folderId),
+              fileFilter =
+                if (data.meta.fileFilter == Glob.all) src.source.fileFilterOrAll
+                else data.meta.fileFilter,
+              tags = (data.meta.tags ++ src.tags.map(_.tagId.id)).distinct
             ),
-            priority = src.priority
+            priority = src.source.priority
           )
-          accId = AccountId(src.cid, src.sid)
+          accId = AccountId(src.source.cid, src.source.sid)
           result <- OptionT.liftF(submit(updata, accId, notifyJoex, itemId))
         } yield result).getOrElse(UploadResult.noSource)
 

modules/common/src/main/scala/docspell/common/Glob.scala

@@ -0,0 +1,168 @@
+package docspell.common
+
+import cats.data.NonEmptyList
+import cats.implicits._
+
+import io.circe.{Decoder, Encoder}
+
+trait Glob {
+
+  /** Matches the input string against this glob. */
+  def matches(in: String): Boolean
+
+  /** If this glob consists of multiple segments, it is the same as
+    * `matches`. If it is only a single segment, it is matched against
+    * the last segment of the input string that is assumed to be a
+    * pathname separated by slash.
+    *
+    * Example:
+    *  test.*  <>  "/a/b/test.txt" => true
+    *  /test.* <>  "/a/b/test.txt" => false
+    */
+  def matchFilenameOrPath(in: String): Boolean
+
+  def asString: String
+}
+
+object Glob {
+  private val separator = '/'
+  private val anyChar   = '|'
+
+  val all = new Glob {
+    def matches(in: String)             = true
+    def matchFilenameOrPath(in: String) = true
+    val asString                        = "*"
+  }
+
+  def pattern(pattern: Pattern): Glob =
+    PatternGlob(pattern)
+
+  /** A simple glob supporting `*` and `?`. */
+  final private case class PatternGlob(pattern: Pattern) extends Glob {
+    def matches(in: String): Boolean =
+      pattern.parts
+        .zipWith(Glob.split(in, Glob.separator))(_.matches(_))
+        .forall(identity)
+
+    def matchFilenameOrPath(in: String): Boolean =
+      if (pattern.parts.tail.isEmpty) matches(split(in, separator).last)
+      else matches(in)
+
+    def asString: String =
+      pattern.asString
+  }
+
+  final private case class AnyGlob(globs: NonEmptyList[Glob]) extends Glob {
+    def matches(in: String) =
+      globs.exists(_.matches(in))
+    def matchFilenameOrPath(in: String) =
+      globs.exists(_.matchFilenameOrPath(in))
+    def asString =
+      globs.toList.map(_.asString).mkString(anyChar.toString)
+  }
+
+  def apply(in: String): Glob = {
+    def single(str: String) =
+      PatternGlob(Pattern(split(str, separator).map(makeSegment)))
+
+    if (in == "*") all
+    else
+      split(in, anyChar) match {
+        case NonEmptyList(_, Nil) =>
+          single(in)
+        case nel =>
+          AnyGlob(nel.map(_.trim).map(single))
+      }
+  }
+
+  case class Pattern(parts: NonEmptyList[Segment]) {
+    def asString =
+      parts.map(_.asString).toList.mkString(separator.toString)
+  }
+
+  object Pattern {
+    def apply(s0: Segment, sm: Segment*): Pattern =
+      Pattern(NonEmptyList.of(s0, sm: _*))
+  }
+
+  case class Segment(tokens: NonEmptyList[Token]) {
+    def matches(in: String): Boolean =
+      consume(in).exists(_.isEmpty)
+
+    def consume(in: String): Option[String] =
+      tokens.foldLeft(in.some) { (rem, token) =>
+        rem.flatMap(token.consume)
+      }
+
+    def asString: String =
+      tokens.toList.map(_.asString).mkString
+  }
+  object Segment {
+    def apply(t0: Token, ts: Token*): Segment =
+      Segment(NonEmptyList.of(t0, ts: _*))
+  }
+
+  sealed trait Token {
+    def consume(str: String): Option[String]
+
+    def asString: String
+  }
+  object Token {
+    case class Literal(asString: String) extends Token {
+      def consume(str: String): Option[String] =
+        if (str.startsWith(asString)) str.drop(asString.length).some
+        else None
+    }
+    case class Until(value: String) extends Token {
+      def consume(str: String): Option[String] =
+        if (value.isEmpty) Some("")
+        else
+          str.indexOf(value) match {
+            case -1 => None
+            case n  => str.substring(n + value.length).some
+          }
+      val asString =
+        s"*$value"
+    }
+    case object Single extends Token {
+      def consume(str: String): Option[String] =
+        if (str.isEmpty()) None
+        else Some(str.drop(1))
+
+      val asString = "?"
+    }
+  }
+
+  private def split(str: String, sep: Char): NonEmptyList[String] =
+    NonEmptyList
+      .fromList(str.split(sep).toList)
+      .getOrElse(NonEmptyList.of(str))
+
+  private def makeSegment(str: String): Segment = {
+    def loop(rem: String, res: List[Token]): List[Token] =
+      if (rem.isEmpty) res
+      else
+        rem.charAt(0) match {
+          case '*' =>
+            val stop = rem.drop(1).takeWhile(c => c != '*' && c != '?')
+            loop(rem.drop(1 + stop.length), Token.Until(stop) :: res)
+          case '?' =>
+            loop(rem.drop(1), Token.Single :: res)
+          case _ =>
+            val lit = rem.takeWhile(c => c != '*' && c != '?')
+            loop(rem.drop(lit.length), Token.Literal(lit) :: res)
+        }
+
+    val fixed = str.replaceAll("\\*+", "*")
+    NonEmptyList
+      .fromList(loop(fixed, Nil).reverse)
+      .map(Segment.apply)
+      .getOrElse(Segment(Token.Literal(str)))
+  }
+
+  implicit val jsonEncoder: Encoder[Glob] =
+    Encoder.encodeString.contramap(_.asString)
+
+  implicit val jsonDecoder: Decoder[Glob] =
+    Decoder.decodeString.map(Glob.apply)
+}

modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala

@@ -38,7 +38,9 @@ object ProcessItemArgs {
       sourceAbbrev: String,
       folderId: Option[Ident],
       validFileTypes: Seq[MimeType],
-      skipDuplicate: Boolean
+      skipDuplicate: Boolean,
+      fileFilter: Option[Glob],
+      tags: Option[List[String]]
   )
 
   object ProcessMeta {

modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

@@ -29,7 +29,11 @@ case class ScanMailboxArgs(
     // set the direction when submitting
     direction: Option[Direction],
     // set a folder for items
-    itemFolder: Option[Ident]
+    itemFolder: Option[Ident],
+    // set a filter for files when importing archives
+    fileFilter: Option[Glob],
+    // set a list of tags to apply to new item
+    tags: Option[List[String]]
 )
 
 object ScanMailboxArgs {

modules/common/src/test/scala/docspell/common/GlobTest.scala

@@ -0,0 +1,111 @@
+package docspell.common
+
+import minitest._
+import Glob._
+
+object GlobTest extends SimpleTestSuite {
+
+  test("literals") {
+    assert(Glob.pattern(Pattern(Segment(Token.Literal("hello")))).matches("hello"))
+    assert(!Glob.pattern(Pattern(Segment(Token.Literal("hello")))).matches("hello1"))
+  }
+
+  test("single wildcards 1") {
+    val glob =
+      Glob.pattern(
+        Pattern(Segment(Token.Literal("s"), Token.Until("p"), Token.Until("t")))
+      )
+
+    assert(glob.matches("snapshot"))
+    assert(!glob.matches("snapshots"))
+  }
+
+  test("single wildcards 2") {
+    val glob =
+      Glob.pattern(Pattern(Segment(Token.Literal("test."), Token.Until(""))))
+
+    assert(glob.matches("test.txt"))
+    assert(glob.matches("test.pdf"))
+    assert(glob.matches("test.converted.pdf"))
+    assert(!glob.matches("test1.txt"))
+    assert(!glob.matches("atest.txt"))
+  }
+
+  test("single parsing") {
+    assertEquals(
+      Glob("s*p*t"),
+      Glob.pattern(
+        Pattern(Segment(Token.Literal("s"), Token.Until("p"), Token.Until("t")))
+      )
+    )
+    assertEquals(
+      Glob("s***p*t"),
+      Glob.pattern(
+        Pattern(Segment(Token.Literal("s"), Token.Until("p"), Token.Until("t")))
+      )
+    )
+    assertEquals(
+      Glob("test.*"),
+      Glob.pattern(Pattern(Segment(Token.Literal("test."), Token.Until(""))))
+    )
+    assertEquals(
+      Glob("stop"),
+      Glob.pattern(Pattern(Segment(Token.Literal("stop"))))
+    )
+    assertEquals(
+      Glob("*stop"),
+      Glob.pattern(Pattern(Segment(Token.Until("stop"))))
+    )
+    assertEquals(Glob("*"), Glob.all)
+  }
+
+  test("with splitting") {
+    assert(Glob("a/b/*").matches("a/b/hello"))
+    assert(!Glob("a/b/*").matches("/a/b/hello"))
+    assert(Glob("/a/b/*").matches("/a/b/hello"))
+    assert(!Glob("/a/b/*").matches("a/b/hello"))
+    assert(!Glob("*/a/b/*").matches("a/b/hello"))
+    assert(Glob("*/a/b/*").matches("test/a/b/hello"))
+  }
+
+  test("asString") {
+    assertEquals(Glob("test.*").asString, "test.*")
+    assertEquals(Glob("s***p*t").asString, "s*p*t")
+    assertEquals(Glob("stop").asString, "stop")
+    assertEquals(Glob("*stop").asString, "*stop")
+    assertEquals(Glob("/a/b/*").asString, "/a/b/*")
+    assertEquals(Glob("*").asString, "*")
+    assertEquals(Glob.all.asString, "*")
+  }
+
+  test("simple matches") {
+    assert(Glob("/test.*").matches("/test.pdf"))
+    assert(!Glob("/test.*").matches("test.pdf"))
+    assert(!Glob("test.*").matches("/test.pdf"))
+  }
+
+  test("matchFilenameOrPath") {
+    assert(Glob("test.*").matchFilenameOrPath("/a/b/test.pdf"))
+    assert(!Glob("/test.*").matchFilenameOrPath("/a/b/test.pdf"))
+    assert(Glob("s*p*t").matchFilenameOrPath("snapshot"))
+    assert(Glob("s*p*t").matchFilenameOrPath("/tmp/snapshot"))
+    assert(Glob("/tmp/s*p*t").matchFilenameOrPath("/tmp/snapshot"))
+
+    assert(Glob("a/b/*").matchFilenameOrPath("a/b/hello"))
+    assert(!Glob("a/b/*").matchFilenameOrPath("/a/b/hello"))
+    assert(Glob("/a/b/*").matchFilenameOrPath("/a/b/hello"))
+    assert(!Glob("/a/b/*").matchFilenameOrPath("a/b/hello"))
+    assert(!Glob("*/a/b/*").matchFilenameOrPath("a/b/hello"))
+    assert(Glob("*/a/b/*").matchFilenameOrPath("test/a/b/hello"))
+  }
+
+  test("anyglob") {
+    assert(Glob("*.pdf|*.txt").matches("test.pdf"))
+    assert(Glob("*.pdf|*.txt").matches("test.txt"))
+    assert(!Glob("*.pdf|*.txt").matches("test.xls"))
+    assert(Glob("*.pdf | *.txt").matches("test.pdf"))
+    assert(Glob("*.pdf | mail.html").matches("test.pdf"))
+    assert(Glob("*.pdf | mail.html").matches("mail.html"))
+    assert(!Glob("*.pdf | mail.html").matches("test.docx"))
+  }
+}

modules/files/src/main/scala/docspell/files/Zip.scala

@@ -9,24 +9,33 @@ import cats.implicits._
 import fs2.{Pipe, Stream}
 
 import docspell.common.Binary
+import docspell.common.Glob
 
 object Zip {
 
   def unzipP[F[_]: ConcurrentEffect: ContextShift](
       chunkSize: Int,
-      blocker: Blocker
+      blocker: Blocker,
+      glob: Glob
   ): Pipe[F, Byte, Binary[F]] =
-    s => unzip[F](chunkSize, blocker)(s)
+    s => unzip[F](chunkSize, blocker, glob)(s)
 
-  def unzip[F[_]: ConcurrentEffect: ContextShift](chunkSize: Int, blocker: Blocker)(
+  def unzip[F[_]: ConcurrentEffect: ContextShift](
+      chunkSize: Int,
+      blocker: Blocker,
+      glob: Glob
+  )(
       data: Stream[F, Byte]
   ): Stream[F, Binary[F]] =
-    data.through(fs2.io.toInputStream[F]).flatMap(in => unzipJava(in, chunkSize, blocker))
+    data
+      .through(fs2.io.toInputStream[F])
+      .flatMap(in => unzipJava(in, chunkSize, blocker, glob))
 
   def unzipJava[F[_]: Sync: ContextShift](
       in: InputStream,
       chunkSize: Int,
-      blocker: Blocker
+      blocker: Blocker,
+      glob: Glob
   ): Stream[F, Binary[F]] = {
     val zin = new ZipInputStream(in)
 
@@ -39,6 +48,7 @@ object Zip {
       .resource(nextEntry)
       .repeat
       .unNoneTerminate
+      .filter(ze => glob.matchFilenameOrPath(ze.getName()))
       .map { ze =>
         val name = Paths.get(ze.getName()).getFileName.toString
         val data =

modules/files/src/test/scala/docspell/files/ZipTest.scala

@@ -4,6 +4,7 @@ import minitest._
 import cats.effect._
 import cats.implicits._
 import scala.concurrent.ExecutionContext
+import docspell.common.Glob
 
 object ZipTest extends SimpleTestSuite {
 
@@ -12,7 +13,7 @@ object ZipTest extends SimpleTestSuite {
 
   test("unzip") {
     val zipFile = ExampleFiles.letters_zip.readURL[IO](8192, blocker)
-    val uncomp  = zipFile.through(Zip.unzip(8192, blocker))
+    val uncomp  = zipFile.through(Zip.unzip(8192, blocker, Glob.all))
 
     uncomp
       .evalMap { entry =>

modules/joex/src/main/scala/docspell/joex/mail/ReadMail.scala

@@ -16,9 +16,10 @@ import emil.{MimeType => _, _}
 object ReadMail {
 
   def readBytesP[F[_]: ConcurrentEffect: ContextShift](
-      logger: Logger[F]
+      logger: Logger[F],
+      glob: Glob
   ): Pipe[F, Byte, Binary[F]] =
-    _.through(bytesToMail(logger)).flatMap(mailToEntries[F](logger))
+    _.through(bytesToMail(logger)).flatMap(mailToEntries[F](logger, glob))
 
   def bytesToMail[F[_]: Sync](logger: Logger[F]): Pipe[F, Byte, Mail[F]] =
     s =>
@@ -26,7 +27,8 @@ object ReadMail {
         s.through(Mail.readBytes[F])
 
   def mailToEntries[F[_]: ConcurrentEffect: ContextShift](
-      logger: Logger[F]
+      logger: Logger[F],
+      glob: Glob
   )(mail: Mail[F]): Stream[F, Binary[F]] = {
     val bodyEntry: F[Option[Binary[F]]] =
       if (mail.body.isEmpty) (None: Option[Binary[F]]).pure[F]
@@ -48,10 +50,12 @@ object ReadMail {
     ) >>
       (Stream
         .eval(bodyEntry)
-        .flatMap(e => Stream.emits(e.toSeq)) ++
+        .flatMap(e => Stream.emits(e.toSeq))
+        .filter(a => glob.matches(a.name)) ++
         Stream
           .eval(TnefExtract.replace(mail))
           .flatMap(m => Stream.emits(m.attachments.all))
+          .filter(a => a.filename.exists(glob.matches))
           .map(a =>
             Binary(a.filename.getOrElse("noname"), a.mimeType.toLocal, a.content)
           ))

modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala

@@ -95,12 +95,12 @@ object ExtractArchive {
       case MimeType.ZipMatch(_) if ra.name.exists(_.endsWith(".zip")) =>
         ctx.logger.info(s"Extracting zip archive ${ra.name.getOrElse("<noname>")}.") *>
           extractZip(ctx, archive)(ra, pos)
-            .flatTap(_ => cleanupParents(ctx, ra, archive))
+            .flatMap(cleanupParents(ctx, ra, archive))
 
       case MimeType.EmailMatch(_) =>
         ctx.logger.info(s"Reading e-mail ${ra.name.getOrElse("<noname>")}") *>
           extractMail(ctx, archive)(ra, pos)
-            .flatTap(_ => cleanupParents(ctx, ra, archive))
+            .flatMap(cleanupParents(ctx, ra, archive))
 
       case _ =>
         ctx.logger.debug(s"Not an archive: ${mime.asString}") *>
@@ -111,7 +111,7 @@ object ExtractArchive {
       ctx: Context[F, _],
       ra: RAttachment,
       archive: Option[RAttachmentArchive]
-  ): F[Unit] =
+  )(extracted: Extracted): F[Extracted] =
     archive match {
       case Some(_) =>
         for {
@@ -121,36 +121,37 @@ object ExtractArchive {
           _ <- ctx.store.transact(RAttachmentArchive.delete(ra.id))
           _ <- ctx.store.transact(RAttachment.delete(ra.id))
           _ <- ctx.store.bitpeace.delete(ra.fileId.id).compile.drain
-        } yield ()
+        } yield extracted
       case None =>
         for {
           _ <- ctx.logger.debug(
             s"Extracted attachment ${ra.name}. Remove it from the item."
           )
           _ <- ctx.store.transact(RAttachment.delete(ra.id))
-        } yield ()
+        } yield extracted.copy(files = extracted.files.filter(_.id != ra.id))
     }
 
   def extractZip[F[_]: ConcurrentEffect: ContextShift](
-      ctx: Context[F, _],
+      ctx: Context[F, ProcessItemArgs],
       archive: Option[RAttachmentArchive]
   )(ra: RAttachment, pos: Int): F[Extracted] = {
     val zipData = ctx.store.bitpeace
       .get(ra.fileId.id)
       .unNoneTerminate
       .through(ctx.store.bitpeace.fetchData2(RangeDef.all))
-
+    val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all)
-    zipData
+    ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *>
-      .through(Zip.unzipP[F](8192, ctx.blocker))
+      zipData
-      .zipWithIndex
+        .through(Zip.unzipP[F](8192, ctx.blocker, glob))
-      .flatMap(handleEntry(ctx, ra, pos, archive, None))
+        .zipWithIndex
-      .foldMonoid
+        .flatMap(handleEntry(ctx, ra, pos, archive, None))
-      .compile
+        .foldMonoid
-      .lastOrError
+        .compile
+        .lastOrError
   }
 
   def extractMail[F[_]: ConcurrentEffect: ContextShift](
-      ctx: Context[F, _],
+      ctx: Context[F, ProcessItemArgs],
       archive: Option[RAttachmentArchive]
   )(ra: RAttachment, pos: Int): F[Extracted] = {
     val email: Stream[F, Byte] = ctx.store.bitpeace
@@ -158,24 +159,26 @@ object ExtractArchive {
       .unNoneTerminate
       .through(ctx.store.bitpeace.fetchData2(RangeDef.all))
 
-    email
+    val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all)
-      .through(ReadMail.bytesToMail[F](ctx.logger))
+    ctx.logger.debug(s"Filtering email attachments with '${glob.asString}'") *>
-      .flatMap { mail =>
+      email
-        val mId = mail.header.messageId
+        .through(ReadMail.bytesToMail[F](ctx.logger))
-        val givenMeta =
+        .flatMap { mail =>
-          for {
+          val mId = mail.header.messageId
-            _ <- ctx.logger.debug(s"Use mail date for item date: ${mail.header.date}")
+          val givenMeta =
-            s <- Sync[F].delay(extractMailMeta(mail))
+            for {
-          } yield s
+              _ <- ctx.logger.debug(s"Use mail date for item date: ${mail.header.date}")
+              s <- Sync[F].delay(extractMailMeta(mail))
+            } yield s
 
-        ReadMail
+          ReadMail
-          .mailToEntries(ctx.logger)(mail)
+            .mailToEntries(ctx.logger, glob)(mail)
-          .zipWithIndex
+            .zipWithIndex
-          .flatMap(handleEntry(ctx, ra, pos, archive, mId)) ++ Stream.eval(givenMeta)
+            .flatMap(handleEntry(ctx, ra, pos, archive, mId)) ++ Stream.eval(givenMeta)
-      }
+        }
-      .foldMonoid
+        .foldMonoid
-      .compile
+        .compile
-      .lastOrError
+        .lastOrError
   }
 
   def extractMailMeta[F[_]](mail: Mail[F]): Extracted =
@@ -239,6 +242,9 @@ object ExtractArchive {
         positions ++ e.positions
       )
 
+    def filterNames(filter: Glob): Extracted =
+      copy(files = files.filter(ra => filter.matches(ra.name.getOrElse(""))))
+
     def setMeta(m: MetaProposal): Extracted =
       setMeta(MetaProposalList.of(m))
 

modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala

@@ -25,6 +25,7 @@ object ProcessItem {
       .flatMap(LinkProposal[F])
       .flatMap(SetGivenData[F](itemOps))
       .flatMap(Task.setProgress(99))
+      .flatMap(RemoveEmptyItem(itemOps))
 
   def processAttachments[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config,

modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala

@@ -91,7 +91,9 @@ object ReProcessItem {
               "",   //source-id
               None, //folder
               Seq.empty,
-              false
+              false,
+              None,
+              None
             ),
             Nil
           ).pure[F]

modules/joex/src/main/scala/docspell/joex/process/RemoveEmptyItem.scala

@@ -0,0 +1,26 @@
+package docspell.joex.process
+
+import cats.effect._
+import cats.implicits._
+
+import docspell.backend.ops.OItem
+import docspell.common._
+import docspell.joex.scheduler.Task
+
+object RemoveEmptyItem {
+
+  def apply[F[_]: Sync](
+      ops: OItem[F]
+  )(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
+    if (data.item.state.isInvalid && data.attachments.isEmpty)
+      Task { ctx =>
+        for {
+          _ <- ctx.logger.warn(s"Removing item as it doesn't have any attachments!")
+          n <- ops.deleteItem(data.item.id, data.item.cid)
+          _ <- ctx.logger.warn(s"Removed item ($n). No item has been created!")
+        } yield data
+      }
+    else
+      Task.pure(data)
+
+}

modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala

@@ -45,9 +45,10 @@ object SetGivenData {
     Task { ctx =>
       val itemId     = data.item.id
       val collective = ctx.args.meta.collective
+      val tags       = (ctx.args.meta.tags.getOrElse(Nil) ++ data.tags).distinct
       for {
-        _ <- ctx.logger.info(s"Set tags from given data: ${data.tags}")
+        _ <- ctx.logger.info(s"Set tags from given data: ${tags}")
-        e <- ops.linkTags(itemId, data.tags, collective).attempt
+        e <- ops.linkTags(itemId, tags, collective).attempt
         _ <- e.fold(
           ex => ctx.logger.warn(s"Error setting tags: ${ex.getMessage}"),
           _ => ().pure[F]

modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala

@@ -47,7 +47,9 @@ object TextExtraction {
         _   <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)).toSeq: _*)
         dur <- start
         _   <- ctx.logger.info(s"Text extraction finished in ${dur.formatExact}")
-      } yield item.copy(metas = txt.map(_.am), tags = txt.flatMap(_.tags).distinct.toList)
+      } yield item
+        .copy(metas = txt.map(_.am))
+        .appendTags(txt.flatMap(_.tags).distinct.toList)
     }
 
   // --  helpers

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -255,7 +255,9 @@ object ScanMailboxTask {
           s"mailbox-${ctx.args.account.user.id}",
           args.itemFolder,
           Seq.empty,
-          true
+          true,
+          args.fileFilter.getOrElse(Glob.all),
+          args.tags.getOrElse(Nil)
         )
         data = OUpload.UploadData(
           multiple = false,

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -1211,7 +1211,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
       responses:
         200:
           description: Ok
@@ -1231,7 +1231,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/Source"
+              $ref: "#/components/schemas/SourceTagIn"
       responses:
         200:
           description: Ok
@@ -3499,6 +3499,14 @@ components:
             The folder id that is applied to items resulting from
             importing mails. If the folder id is not valid when the
             task executes, items have no folder set.
+        tags:
+          $ref: "#/components/schemas/StringList"
+        fileFilter:
+          description: |
+            A glob to filter attachments to import.
+          type: string
+          format: glob
+
     ImapSettingsList:
       description: |
         A list of user email settings.
@@ -4256,6 +4264,18 @@ components:
         A folderId can be given, the item is placed into this folder
         after creation.
 
+        The `fileFilter` is an optional glob for filtering files to
+        import. Only applicable if archive files are uploaded. It
+        applies to all of them. For example, to only import pdf files
+        when uploading e-mails, use `*.pdf`. If the pattern doesn't
+        contain a slash `/`, then it is applied to all file names.
+        Otherwise it is applied to the complete path in the archive
+        (useful for zip files). Note that the archive file itself is
+        always saved completely, too.
+
+        The `tags` input allows to provide tags that should be applied
+        to the item being created. This only works if the tags already
+        exist. It is possible to specify their ids or names.
       required:
         - multiple
       properties:
@@ -4271,6 +4291,11 @@ components:
         skipDuplicates:
           type: boolean
           default: false
+        tags:
+          $ref: "#/components/schemas/StringList"
+        fileFilter:
+          type: string
+          format: glob
 
     Collective:
       description: |
@@ -4341,7 +4366,7 @@ components:
         items:
           type: array
           items:
-            $ref: "#/components/schemas/Source"
+            $ref: "#/components/schemas/SourceAndTags"
     Source:
       description: |
         Data about a Source. A source defines the endpoint where
@@ -4375,10 +4400,38 @@ components:
         folder:
           type: string
           format: ident
+        fileFilter:
+          type: string
+          format: glob
         created:
           description: DateTime
           type: integer
           format: date-time
+    SourceTagIn:
+      description: |
+        A source and optional tags (ids or names) for updating/adding.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          type: array
+          items:
+            type: string
+    SourceAndTags:
+      description: |
+        A source and optional tags.
+      required:
+        - source
+        - tags
+      properties:
+        source:
+          $ref: "#/components/schema/Source"
+        tags:
+          $ref: "#/components/schema/TagList"
+
     EquipmentList:
       description: |
         A list of equipments.

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -310,13 +310,16 @@ trait Conversions {
               sourceName,
               m.folder,
               validFileTypes,
-              m.skipDuplicates.getOrElse(false)
+              m.skipDuplicates.getOrElse(false),
+              m.fileFilter.getOrElse(Glob.all),
+              m.tags.map(_.items).getOrElse(Nil)
             )
           )
         )
       )
       .getOrElse(
-        (true, UploadMeta(None, sourceName, None, validFileTypes, false)).pure[F]
+        (true, UploadMeta(None, sourceName, None, validFileTypes, false, Glob.all, Nil))
+          .pure[F]
       )
 
     val files = mp.parts
@@ -521,21 +524,36 @@ trait Conversions {
 
   // sources
 
-  def mkSource(s: RSource): Source =
+  def mkSource(s: SourceData): SourceAndTags =
-    Source(
+    SourceAndTags(
-      s.sid,
+      Source(
-      s.abbrev,
+        s.source.sid,
-      s.description,
+        s.source.abbrev,
-      s.counter,
+        s.source.description,
-      s.enabled,
+        s.source.counter,
-      s.priority,
+        s.source.enabled,
-      s.folderId,
+        s.source.priority,
-      s.created
+        s.source.folderId,
+        s.source.fileFilter,
+        s.source.created
+      ),
+      TagList(s.tags.length, s.tags.map(mkTag).toList)
     )
 
   def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
     timeId.map({ case (id, now) =>
-      RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder)
+      RSource(
+        id,
+        cid,
+        s.abbrev,
+        s.description,
+        0,
+        s.enabled,
+        s.priority,
+        now,
+        s.folder,
+        s.fileFilter
+      )
     })
 
   def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
@@ -548,7 +566,8 @@ trait Conversions {
       s.enabled,
       s.priority,
       s.created,
-      s.folder
+      s.folder,
+      s.fileFilter
     )
 
   // equipment

modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala

@@ -113,7 +113,9 @@ object ScanMailboxRoutes {
           settings.targetFolder,
           settings.deleteMail,
           settings.direction,
-          settings.itemFolder
+          settings.itemFolder,
+          settings.fileFilter,
+          settings.tags.map(_.items)
         )
       )
     )
@@ -141,6 +143,8 @@ object ScanMailboxRoutes {
       task.args.targetFolder,
       task.args.deleteMail,
       task.args.direction,
-      task.args.itemFolder
+      task.args.itemFolder,
+      task.args.tags.map(StringList.apply),
+      task.args.fileFilter
     )
 }

modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala

@@ -30,17 +30,17 @@ object SourceRoutes {
 
       case req @ POST -> Root =>
         for {
-          data  <- req.as[Source]
+          data  <- req.as[SourceTagIn]
-          src   <- newSource(data, user.account.collective)
+          src   <- newSource(data.source, user.account.collective)
-          added <- backend.source.add(src)
+          added <- backend.source.add(src, data.tags)
           resp  <- Ok(basicResult(added, "Source added."))
         } yield resp
 
       case req @ PUT -> Root =>
         for {
-          data <- req.as[Source]
+          data <- req.as[SourceTagIn]
-          src = changeSource(data, user.account.collective)
+          src = changeSource(data.source, user.account.collective)
-          updated <- backend.source.update(src)
+          updated <- backend.source.update(src, data.tags)
           resp    <- Ok(basicResult(updated, "Source updated."))
         } yield resp
 

modules/store/src/main/resources/db/migration/h2/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/resources/db/migration/mariadb/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE `source`
+ADD COLUMN `file_filter` varchar(254) NULL;
+
+CREATE TABLE `tagsource` (
+  `id` varchar(254) not null primary key,
+  `source_id` varchar(254) not null,
+  `tag_id` varchar(254) not null,
+  unique (`source_id`, `tag_id`),
+  foreign key (`source_id`) references `source`(`sid`),
+  foreign key (`tag_id`) references `tag`(`tid`)
+);

modules/store/src/main/resources/db/migration/postgresql/V1.12.0__upload_data.sql

@@ -0,0 +1,11 @@
+ALTER TABLE "source"
+ADD COLUMN "file_filter" varchar(254) NULL;
+
+CREATE TABLE "tagsource" (
+  "id" varchar(254) not null primary key,
+  "source_id" varchar(254) not null,
+  "tag_id" varchar(254) not null,
+  unique ("source_id", "tag_id"),
+  foreign key ("source_id") references "source"("sid"),
+  foreign key ("tag_id") references "tag"("tid")
+);

modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala

@@ -91,6 +91,9 @@ trait DoobieMeta extends EmilDoobieMeta {
 
   implicit val metaCalEvent: Meta[CalEvent] =
     Meta[String].timap(CalEvent.unsafe)(_.asString)
+
+  implicit val metaGlob: Meta[Glob] =
+    Meta[String].timap(Glob.apply)(_.asString)
 }
 
 object DoobieMeta extends DoobieMeta {

modules/store/src/main/scala/docspell/store/records/RSource.scala

@@ -16,8 +16,13 @@ case class RSource(
     enabled: Boolean,
     priority: Priority,
     created: Timestamp,
-    folderId: Option[Ident]
+    folderId: Option[Ident],
-) {}
+    fileFilter: Option[Glob]
+) {
+
+  def fileFilterOrAll: Glob =
+    fileFilter.getOrElse(Glob.all)
+}
 
 object RSource {
 
@@ -34,9 +39,21 @@ object RSource {
     val priority    = Column("priority")
     val created     = Column("created")
     val folder      = Column("folder_id")
+    val fileFilter  = Column("file_filter")
 
     val all =
-      List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
+      List(
+        sid,
+        cid,
+        abbrev,
+        description,
+        counter,
+        enabled,
+        priority,
+        created,
+        folder,
+        fileFilter
+      )
   }
 
   import Columns._
@@ -45,7 +62,7 @@ object RSource {
     val sql = insertRow(
       table,
       all,
-      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
+      fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId},${v.fileFilter}"
     )
     sql.update.run
   }
@@ -60,7 +77,8 @@ object RSource {
         description.setTo(v.description),
         enabled.setTo(v.enabled),
         priority.setTo(v.priority),
-        folder.setTo(v.folderId)
+        folder.setTo(v.folderId),
+        fileFilter.setTo(v.fileFilter)
       )
     )
     sql.update.run
@@ -83,10 +101,11 @@ object RSource {
     sql.query[Int].unique.map(_ > 0)
   }
 
-  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] = {
+  def findEnabled(id: Ident): ConnectionIO[Option[RSource]] =
-    val sql = selectSimple(all, table, and(sid.is(id), enabled.is(true)))
+    findEnabledSql(id).query[RSource].option
-    sql.query[RSource].option
+
-  }
+  private[records] def findEnabledSql(id: Ident): Fragment =
+    selectSimple(all, table, and(sid.is(id), enabled.is(true)))
 
   def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] =
     selectSimple(List(cid), table, sid.is(sourceId)).query[Ident].option
@@ -94,10 +113,11 @@ object RSource {
   def findAll(
       coll: Ident,
       order: Columns.type => Column
-  ): ConnectionIO[Vector[RSource]] = {
+  ): ConnectionIO[Vector[RSource]] =
-    val sql = selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
+    findAllSql(coll, order).query[RSource].to[Vector]
-    sql.query[RSource].to[Vector]
+
-  }
+  private[records] def findAllSql(coll: Ident, order: Columns.type => Column): Fragment =
+    selectSimple(all, table, cid.is(coll)) ++ orderBy(order(Columns).f)
 
   def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
     deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run

modules/store/src/main/scala/docspell/store/records/RTag.scala

@@ -104,6 +104,18 @@ object RTag {
     ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
   }
 
+  def findBySource(source: Ident): ConnectionIO[Vector[RTag]] = {
+    val rcol = all.map(_.prefix("t"))
+    (selectSimple(
+      rcol,
+      table ++ fr"t," ++ RTagSource.table ++ fr"s",
+      and(
+        RTagSource.Columns.sourceId.prefix("s").is(source),
+        RTagSource.Columns.tagId.prefix("s").is(tid.prefix("t"))
+      )
+    ) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector]
+  }
+
   def findAllByNameOrId(
       nameOrIds: List[String],
       coll: Ident

modules/store/src/main/scala/docspell/store/records/RTagSource.scala

@@ -0,0 +1,56 @@
+package docspell.store.records
+
+import cats.effect.Sync
+import cats.implicits._
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+case class RTagSource(id: Ident, sourceId: Ident, tagId: Ident) {}
+
+object RTagSource {
+
+  val table = fr"tagsource"
+
+  object Columns {
+    val id       = Column("id")
+    val sourceId = Column("source_id")
+    val tagId    = Column("tag_id")
+    val all      = List(id, sourceId, tagId)
+  }
+  import Columns._
+
+  def createNew[F[_]: Sync](source: Ident, tag: Ident): F[RTagSource] =
+    Ident.randomId[F].map(id => RTagSource(id, source, tag))
+
+  def insert(v: RTagSource): ConnectionIO[Int] =
+    insertRow(table, all, fr"${v.id},${v.sourceId},${v.tagId}").update.run
+
+  def deleteSourceTags(source: Ident): ConnectionIO[Int] =
+    deleteFrom(table, sourceId.is(source)).update.run
+
+  def deleteTag(tid: Ident): ConnectionIO[Int] =
+    deleteFrom(table, tagId.is(tid)).update.run
+
+  def findBySource(source: Ident): ConnectionIO[Vector[RTagSource]] =
+    selectSimple(all, table, sourceId.is(source)).query[RTagSource].to[Vector]
+
+  def setAllTags(source: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
+    if (tags.isEmpty) 0.pure[ConnectionIO]
+    else
+      for {
+        entities <- tags.toList.traverse(tagId =>
+          Ident.randomId[ConnectionIO].map(id => RTagSource(id, source, tagId))
+        )
+        n <- insertRows(
+          table,
+          all,
+          entities.map(v => fr"${v.id},${v.sourceId},${v.tagId}")
+        ).update.run
+      } yield n
+
+}

modules/store/src/main/scala/docspell/store/records/SourceData.scala

@@ -0,0 +1,87 @@
+package docspell.store.records
+
+import cats.effect.concurrent.Ref
+import cats.implicits._
+import fs2.Stream
+
+import docspell.common._
+import docspell.store.impl.Implicits._
+import docspell.store.impl._
+
+import doobie._
+import doobie.implicits._
+
+/** Combines a source record (RSource) and a list of associated tags.
+  */
+case class SourceData(source: RSource, tags: Vector[RTag])
+
+object SourceData {
+
+  def fromSource(s: RSource): SourceData =
+    SourceData(s, Vector.empty)
+
+  def findAll(
+      coll: Ident,
+      order: RSource.Columns.type => Column
+  ): Stream[ConnectionIO, SourceData] =
+    findAllWithTags(RSource.findAllSql(coll, order).query[RSource].stream)
+
+  private def findAllWithTags(
+      select: Stream[ConnectionIO, RSource]
+  ): Stream[ConnectionIO, SourceData] = {
+    def findTag(
+        cache: Ref[ConnectionIO, Map[Ident, RTag]],
+        tagSource: RTagSource
+    ): ConnectionIO[Option[RTag]] =
+      for {
+        cc <- cache.get
+        fromCache = cc.get(tagSource.tagId)
+        orFromDB <-
+          if (fromCache.isDefined) fromCache.pure[ConnectionIO]
+          else RTag.findById(tagSource.tagId)
+        _ <-
+          if (fromCache.isDefined) ().pure[ConnectionIO]
+          else
+            orFromDB match {
+              case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
+              case None    => ().pure[ConnectionIO]
+            }
+      } yield orFromDB
+
+    for {
+      resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
+      source       <- select
+      tagSources   <- Stream.eval(RTagSource.findBySource(source.sid))
+      tags         <- Stream.eval(tagSources.traverse(ti => findTag(resolvedTags, ti)))
+    } yield SourceData(source, tags.flatten)
+  }
+
+  def findEnabled(id: Ident): ConnectionIO[Option[SourceData]] =
+    findAllWithTags(RSource.findEnabledSql(id).query[RSource].stream).head.compile.last
+
+  def insert(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.insert(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def update(data: RSource, tags: List[String]): ConnectionIO[Int] =
+    for {
+      n0   <- RSource.updateNoCounter(data)
+      tags <- RTag.findAllByNameOrId(tags, data.cid)
+      _    <- RTagSource.deleteSourceTags(data.sid)
+      n1 <- tags.traverse(tag =>
+        RTagSource.createNew[ConnectionIO](data.sid, tag.tagId).flatMap(RTagSource.insert)
+      )
+    } yield n0 + n1.sum
+
+  def delete(source: Ident, coll: Ident): ConnectionIO[Int] =
+    for {
+      n0 <- RTagSource.deleteSourceTags(source)
+      n1 <- RSource.delete(source, coll)
+    } yield n0 + n1
+
+}

modules/webapp/src/main/elm/Api.elm

@@ -175,8 +175,9 @@ import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
 import Api.Model.ScanMailboxSettingsList exposing (ScanMailboxSettingsList)
 import Api.Model.SentMails exposing (SentMails)
 import Api.Model.SimpleMail exposing (SimpleMail)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Api.Model.StringList exposing (StringList)
 import Api.Model.Tag exposing (Tag)
 import Api.Model.TagCloud exposing (TagCloud)
@@ -1144,17 +1145,22 @@ getSources flags receive =
         }
 
 
-postSource : Flags -> Source -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+postSource : Flags -> SourceAndTags -> (Result Http.Error BasicResult -> msg) -> Cmd msg
 postSource flags source receive =
     let
+        st =
+            { source = source.source
+            , tags = List.map .id source.tags.items
+            }
+
         params =
             { url = flags.config.baseUrl ++ "/api/v1/sec/source"
             , account = getAccount flags
-            , body = Http.jsonBody (Api.Model.Source.encode source)
+            , body = Http.jsonBody (Api.Model.SourceTagIn.encode st)
             , expect = Http.expectJson receive Api.Model.BasicResult.decoder
             }
     in
-    if source.id == "" then
+    if source.source.id == "" then
         Http2.authPost params
 
     else

modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm

@@ -15,6 +15,9 @@ import Api.Model.FolderList exposing (FolderList)
 import Api.Model.IdName exposing (IdName)
 import Api.Model.ImapSettingsList exposing (ImapSettingsList)
 import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
+import Api.Model.StringList exposing (StringList)
+import Api.Model.Tag exposing (Tag)
+import Api.Model.TagList exposing (TagList)
 import Comp.CalEventInput
 import Comp.Dropdown exposing (isDropdownChangeMsg)
 import Comp.IntField
@@ -34,6 +37,7 @@ import Util.Folder exposing (mkFolderOption)
 import Util.Http
 import Util.List
 import Util.Maybe
+import Util.Tag
 import Util.Update
 
 
@@ -56,6 +60,9 @@ type alias Model =
     , folderModel : Comp.Dropdown.Model IdName
     , allFolders : List FolderItem
     , itemFolderId : Maybe String
+    , tagModel : Comp.Dropdown.Model Tag
+    , existingTags : List String
+    , fileFilter : Maybe String
     }
 
 
@@ -84,6 +91,9 @@ type Msg
     | YesNoDeleteMsg Comp.YesNoDimmer.Msg
     | GetFolderResp (Result Http.Error FolderList)
     | FolderDropdownMsg (Comp.Dropdown.Msg IdName)
+    | GetTagResp (Result Http.Error TagList)
+    | TagDropdownMsg (Comp.Dropdown.Msg Tag)
+    | SetFileFilter String
 
 
 initWith : Flags -> ScanMailboxSettings -> ( Model, Cmd Msg )
@@ -120,12 +130,18 @@ initWith flags s =
         , formMsg = Nothing
         , yesNoDelete = Comp.YesNoDimmer.emptyModel
         , itemFolderId = s.itemFolder
+        , tagModel = Util.Tag.makeDropdownModel
+        , existingTags =
+            Maybe.map .items s.tags
+                |> Maybe.withDefault []
+        , fileFilter = s.fileFilter
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
         , nc
         , Cmd.map CalEventMsg sc
         , Api.getFolders flags "" False GetFolderResp
+        , Api.getTags flags "" GetTagResp
         ]
     )
 
@@ -134,7 +150,7 @@ init : Flags -> ( Model, Cmd Msg )
 init flags =
     let
         initialSchedule =
-            Data.Validated.Unknown Data.CalEvent.everyMonth
+            Data.Validated.Valid Data.CalEvent.everyMonth
 
         sm =
             Comp.CalEventInput.initDefault
@@ -156,7 +172,7 @@ init flags =
       , schedule = initialSchedule
       , scheduleModel = sm
       , formMsg = Nothing
-      , loading = 2
+      , loading = 3
       , yesNoDelete = Comp.YesNoDimmer.emptyModel
       , folderModel =
             Comp.Dropdown.makeSingle
@@ -165,10 +181,14 @@ init flags =
                 }
       , allFolders = []
       , itemFolderId = Nothing
+      , tagModel = Util.Tag.makeDropdownModel
+      , existingTags = []
+      , fileFilter = Nothing
       }
     , Cmd.batch
         [ Api.getImapSettings flags "" ConnResp
         , Api.getFolders flags "" False GetFolderResp
+        , Api.getTags flags "" GetTagResp
         ]
     )
 
@@ -196,9 +216,9 @@ makeSettings model =
             else
                 Valid model.folders
 
-        make smtp timer folders =
+        make imap timer folders =
             { prev
-                | imapConnection = smtp
+                | imapConnection = imap
                 , enabled = model.enabled
                 , receivedSinceHours = model.receivedHours
                 , deleteMail = model.deleteMail
@@ -207,6 +227,16 @@ makeSettings model =
                 , direction = Maybe.map Data.Direction.toString model.direction
                 , schedule = Data.CalEvent.makeEvent timer
                 , itemFolder = model.itemFolderId
+                , fileFilter = model.fileFilter
+                , tags =
+                    case Comp.Dropdown.getSelected model.tagModel of
+                        [] ->
+                            Nothing
+
+                        els ->
+                            List.map .id els
+                                |> StringList
+                                |> Just
             }
     in
     Data.Validated.map3 make
@@ -501,6 +531,61 @@ update flags msg model =
             in
             ( model_, NoAction, Cmd.map FolderDropdownMsg c2 )
 
+        GetTagResp (Ok list) ->
+            let
+                contains el =
+                    List.member el model.existingTags
+
+                isExistingTag t =
+                    contains t.id || contains t.name
+
+                selected =
+                    List.filter isExistingTag list.items
+                        |> Comp.Dropdown.SetSelection
+
+                opts =
+                    Comp.Dropdown.SetOptions list.items
+
+                ( tagModel_, tagcmd ) =
+                    Util.Update.andThen1
+                        [ Comp.Dropdown.update selected
+                        , Comp.Dropdown.update opts
+                        ]
+                        model.tagModel
+
+                nextModel =
+                    { model
+                        | loading = model.loading - 1
+                        , tagModel = tagModel_
+                    }
+            in
+            ( nextModel
+            , NoAction
+            , Cmd.map TagDropdownMsg tagcmd
+            )
+
+        GetTagResp (Err _) ->
+            ( { model | loading = model.loading - 1 }
+            , NoAction
+            , Cmd.none
+            )
+
+        TagDropdownMsg lm ->
+            let
+                ( m2, c2 ) =
+                    Comp.Dropdown.update lm model.tagModel
+
+                newModel =
+                    { model | tagModel = m2 }
+            in
+            ( newModel, NoAction, Cmd.map TagDropdownMsg c2 )
+
+        SetFileFilter str ->
+            ( { model | fileFilter = Util.Maybe.fromString str }
+            , NoAction
+            , Cmd.none
+            )
+
 
 
 --- View
@@ -603,6 +688,9 @@ view extraClasses settings model =
                 , text " is not set."
                 ]
             ]
+        , div [ class "ui dividing header" ]
+            [ text "Metadata"
+            ]
         , div [ class "required field" ]
             [ label [] [ text "Item direction" ]
             , div [ class "grouped fields" ]
@@ -668,6 +756,49 @@ disappear then.
                       """
                 ]
             ]
+        , div [ class "field" ]
+            [ label [] [ text "Tags" ]
+            , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
+            , div [ class "small-info" ]
+                [ text "Choose tags that should be applied to items."
+                ]
+            ]
+        , div
+            [ class "field"
+            ]
+            [ label [] [ text "File Filter" ]
+            , input
+                [ type_ "text"
+                , onInput SetFileFilter
+                , placeholder "File Filter"
+                , model.fileFilter
+                    |> Maybe.withDefault ""
+                    |> value
+                ]
+                []
+            , div [ class "small-info" ]
+                [ text "Specify a file glob to filter attachments. For example, to only extract pdf files: "
+                , code []
+                    [ text "*.pdf"
+                    ]
+                , text ". If you want to include the mail body, allow html files or "
+                , code []
+                    [ text "mail.html"
+                    ]
+                , text ". Globs can be combined via OR, like this: "
+                , code []
+                    [ text "*.pdf|mail.html"
+                    ]
+                , text "No file filter defaults to "
+                , code []
+                    [ text "*"
+                    ]
+                , text " that includes all"
+                ]
+            ]
+        , div [ class "ui dividing header" ]
+            [ text "Schedule"
+            ]
         , div [ class "required field" ]
             [ label []
                 [ text "Schedule"

modules/webapp/src/main/elm/Comp/SourceForm.elm

@@ -12,7 +12,9 @@ import Api
 import Api.Model.FolderItem exposing (FolderItem)
 import Api.Model.FolderList exposing (FolderList)
 import Api.Model.IdName exposing (IdName)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
+import Api.Model.Tag exposing (Tag)
+import Api.Model.TagList exposing (TagList)
 import Comp.Dropdown exposing (isDropdownChangeMsg)
 import Comp.FixedDropdown
 import Data.Flags exposing (Flags)
@@ -24,10 +26,13 @@ import Html.Events exposing (onCheck, onInput)
 import Http
 import Markdown
 import Util.Folder exposing (mkFolderOption)
+import Util.Maybe
+import Util.Tag
+import Util.Update
 
 
 type alias Model =
-    { source : Source
+    { source : SourceAndTags
     , abbrev : String
     , description : Maybe String
     , priorityModel : Comp.FixedDropdown.Model Priority
@@ -36,12 +41,14 @@ type alias Model =
     , folderModel : Comp.Dropdown.Model IdName
     , allFolders : List FolderItem
     , folderId : Maybe String
+    , tagModel : Comp.Dropdown.Model Tag
+    , fileFilter : Maybe String
     }
 
 
 emptyModel : Model
 emptyModel =
-    { source = Api.Model.Source.empty
+    { source = Api.Model.SourceAndTags.empty
     , abbrev = ""
     , description = Nothing
     , priorityModel =
@@ -57,13 +64,18 @@ emptyModel =
             }
     , allFolders = []
     , folderId = Nothing
+    , tagModel = Util.Tag.makeDropdownModel
+    , fileFilter = Nothing
     }
 
 
 init : Flags -> ( Model, Cmd Msg )
 init flags =
     ( emptyModel
-    , Api.getFolders flags "" False GetFolderResp
+    , Cmd.batch
+        [ Api.getFolders flags "" False GetFolderResp
+        , Api.getTags flags "" GetTagResp
+        ]
     )
 
 
@@ -72,29 +84,42 @@ isValid model =
     model.abbrev /= ""
 
 
-getSource : Model -> Source
+getSource : Model -> SourceAndTags
 getSource model =
     let
-        s =
+        st =
             model.source
+
+        s =
+            st.source
+
+        tags =
+            Comp.Dropdown.getSelected model.tagModel
+
+        n =
+            { s
+                | abbrev = model.abbrev
+                , description = model.description
+                , enabled = model.enabled
+                , priority = Data.Priority.toName model.priority
+                , folder = model.folderId
+                , fileFilter = model.fileFilter
+            }
     in
-    { s
+    { st | source = n, tags = TagList (List.length tags) tags }
-        | abbrev = model.abbrev
-        , description = model.description
-        , enabled = model.enabled
-        , priority = Data.Priority.toName model.priority
-        , folder = model.folderId
-    }
 
 
 type Msg
     = SetAbbrev String
-    | SetSource Source
+    | SetSource SourceAndTags
     | SetDescr String
     | ToggleEnabled
     | PrioDropdownMsg (Comp.FixedDropdown.Msg Priority)
     | GetFolderResp (Result Http.Error FolderList)
     | FolderDropdownMsg (Comp.Dropdown.Msg IdName)
+    | GetTagResp (Result Http.Error TagList)
+    | TagDropdownMsg (Comp.Dropdown.Msg Tag)
+    | SetFileFilter String
 
 
 
@@ -106,29 +131,34 @@ update flags msg model =
     case msg of
         SetSource t ->
             let
-                post =
+                stpost =
                     model.source
 
+                post =
+                    stpost.source
+
                 np =
                     { post
-                        | id = t.id
+                        | id = t.source.id
-                        , abbrev = t.abbrev
+                        , abbrev = t.source.abbrev
-                        , description = t.description
+                        , description = t.source.description
-                        , priority = t.priority
+                        , priority = t.source.priority
-                        , enabled = t.enabled
+                        , enabled = t.source.enabled
-                        , folder = t.folder
+                        , folder = t.source.folder
+                        , fileFilter = t.source.fileFilter
                     }
 
                 newModel =
                     { model
-                        | source = np
+                        | source = { stpost | source = np }
-                        , abbrev = t.abbrev
+                        , abbrev = t.source.abbrev
-                        , description = t.description
+                        , description = t.source.description
                         , priority =
-                            Data.Priority.fromString t.priority
+                            Data.Priority.fromString t.source.priority
                                 |> Maybe.withDefault Data.Priority.Low
-                        , enabled = t.enabled
+                        , enabled = t.source.enabled
-                        , folderId = t.folder
+                        , folderId = t.source.folder
+                        , fileFilter = t.source.fileFilter
                     }
 
                 mkIdName id =
@@ -143,14 +173,21 @@ update flags msg model =
                         model.allFolders
 
                 sel =
-                    case Maybe.map mkIdName t.folder of
+                    case Maybe.map mkIdName t.source.folder of
                         Just idref ->
                             idref
 
                         Nothing ->
                             []
+
+                tags =
+                    Comp.Dropdown.SetSelection t.tags.items
             in
-            update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel)) newModel
+            Util.Update.andThen1
+                [ update flags (FolderDropdownMsg (Comp.Dropdown.SetSelection sel))
+                , update flags (TagDropdownMsg tags)
+                ]
+                newModel
 
         ToggleEnabled ->
             ( { model | enabled = not model.enabled }, Cmd.none )
@@ -159,14 +196,7 @@ update flags msg model =
             ( { model | abbrev = n }, Cmd.none )
 
         SetDescr d ->
-            ( { model
+            ( { model | description = Util.Maybe.fromString d }
-                | description =
-                    if d /= "" then
-                        Just d
-
-                    else
-                        Nothing
-              }
             , Cmd.none
             )
 
@@ -226,6 +256,31 @@ update flags msg model =
             in
             ( model_, Cmd.map FolderDropdownMsg c2 )
 
+        GetTagResp (Ok list) ->
+            let
+                opts =
+                    Comp.Dropdown.SetOptions list.items
+            in
+            update flags (TagDropdownMsg opts) model
+
+        GetTagResp (Err _) ->
+            ( model, Cmd.none )
+
+        TagDropdownMsg lm ->
+            let
+                ( m2, c2 ) =
+                    Comp.Dropdown.update lm model.tagModel
+
+                newModel =
+                    { model | tagModel = m2 }
+            in
+            ( newModel, Cmd.map TagDropdownMsg c2 )
+
+        SetFileFilter d ->
+            ( { model | fileFilter = Util.Maybe.fromString d }
+            , Cmd.none
+            )
+
 
 
 --- View
@@ -260,6 +315,7 @@ view flags settings model =
             , textarea
                 [ onInput SetDescr
                 , model.description |> Maybe.withDefault "" |> value
+                , rows 3
                 ]
                 []
             ]
@@ -281,12 +337,26 @@ view flags settings model =
                     (Just priorityItem)
                     model.priorityModel
                 )
+            , div [ class "small-info" ]
+                [ text "The priority used by the scheduler when processing uploaded files."
+                ]
+            ]
+        , div [ class "ui dividing header" ]
+            [ text "Metadata"
+            ]
+        , div [ class "ui message" ]
+            [ text "Metadata specified here is automatically attached to each item uploaded "
+            , text "through this source, unless it is overriden in the upload request meta data. "
+            , text "Tags from the request are added to those defined here."
             ]
         , div [ class "field" ]
             [ label []
                 [ text "Folder"
                 ]
             , Html.map FolderDropdownMsg (Comp.Dropdown.view settings model.folderModel)
+            , div [ class "small-info" ]
+                [ text "Choose a folder to automatically put items into."
+                ]
             , div
                 [ classList
                     [ ( "ui warning message", True )
@@ -301,6 +371,38 @@ disappear then.
                       """
                 ]
             ]
+        , div [ class "field" ]
+            [ label [] [ text "Tags" ]
+            , Html.map TagDropdownMsg (Comp.Dropdown.view settings model.tagModel)
+            , div [ class "small-info" ]
+                [ text "Choose tags that should be applied to items."
+                ]
+            ]
+        , div
+            [ class "field"
+            ]
+            [ label [] [ text "File Filter" ]
+            , input
+                [ type_ "text"
+                , onInput SetFileFilter
+                , placeholder "File Filter"
+                , model.fileFilter
+                    |> Maybe.withDefault ""
+                    |> value
+                ]
+                []
+            , div [ class "small-info" ]
+                [ text "Specify a file glob to filter files when uploading archives "
+                , text "(e.g. for email and zip). For example, to only extract pdf files: "
+                , code []
+                    [ text "*.pdf"
+                    ]
+                , text ". Globs can be combined via OR, like this: "
+                , code []
+                    [ text "*.pdf|mail.html"
+                    ]
+                ]
+            ]
         ]
 
 

modules/webapp/src/main/elm/Comp/SourceManage.elm

@@ -8,8 +8,9 @@ module Comp.SourceManage exposing
 
 import Api
 import Api.Model.BasicResult exposing (BasicResult)
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Api.Model.SourceList exposing (SourceList)
+import Api.Model.SourceTagIn exposing (SourceTagIn)
 import Comp.SourceForm
 import Comp.SourceTable exposing (SelectMode(..))
 import Comp.YesNoDimmer
@@ -31,7 +32,7 @@ type alias Model =
     , formError : Maybe String
     , loading : Bool
     , deleteConfirm : Comp.YesNoDimmer.Model
-    , sources : List Source
+    , sources : List SourceAndTags
     }
 
 
@@ -145,7 +146,7 @@ update flags msg model =
         InitNewSource ->
             let
                 source =
-                    Api.Model.Source.empty
+                    Api.Model.SourceAndTags.empty
 
                 nm =
                     { model | viewMode = Edit source, formError = Nothing }
@@ -196,7 +197,7 @@ update flags msg model =
 
                 cmd =
                     if confirmed then
-                        Api.deleteSource flags src.id SubmitResp
+                        Api.deleteSource flags src.source.id SubmitResp
 
                     else
                         Cmd.none
@@ -248,22 +249,22 @@ viewTable model =
         ]
 
 
-viewLinks : Flags -> UiSettings -> Source -> Html Msg
+viewLinks : Flags -> UiSettings -> SourceAndTags -> Html Msg
 viewLinks flags _ source =
     let
         appUrl =
-            flags.config.baseUrl ++ "/app/upload/" ++ source.id
+            flags.config.baseUrl ++ "/app/upload/" ++ source.source.id
 
         apiUrl =
-            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.id
+            flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ source.source.id
     in
     div
         []
         [ h3 [ class "ui dividing header" ]
             [ text "Public Uploads: "
-            , text source.abbrev
+            , text source.source.abbrev
             , div [ class "sub header" ]
-                [ text source.id
+                [ text source.source.id
                 ]
             ]
         , p []
@@ -273,7 +274,7 @@ viewLinks flags _ source =
             ]
         , p []
             [ text "There have been "
-            , String.fromInt source.counter |> text
+            , String.fromInt source.source.counter |> text
             , text " items created through this source."
             ]
         , h4 [ class "ui header" ]
@@ -358,7 +359,7 @@ viewForm : Flags -> UiSettings -> Model -> List (Html Msg)
 viewForm flags settings model =
     let
         newSource =
-            model.formModel.source.id == ""
+            model.formModel.source.source.id == ""
     in
     [ if newSource then
         h3 [ class "ui top attached header" ]
@@ -367,10 +368,10 @@ viewForm flags settings model =
 
       else
         h3 [ class "ui top attached header" ]
-            [ text ("Edit: " ++ model.formModel.source.abbrev)
+            [ text ("Edit: " ++ model.formModel.source.source.abbrev)
             , div [ class "sub header" ]
                 [ text "Id: "
-                , text model.formModel.source.id
+                , text model.formModel.source.source.id
                 ]
             ]
     , Html.form [ class "ui attached segment", onSubmit Submit ]

modules/webapp/src/main/elm/Comp/SourceTable.elm

@@ -6,7 +6,7 @@ module Comp.SourceTable exposing
     , view
     )
 
-import Api.Model.Source exposing (Source)
+import Api.Model.SourceAndTags exposing (SourceAndTags)
 import Data.Flags exposing (Flags)
 import Data.Priority
 import Html exposing (..)
@@ -15,8 +15,8 @@ import Html.Events exposing (onClick)
 
 
 type SelectMode
-    = Edit Source
+    = Edit SourceAndTags
-    | Display Source
+    | Display SourceAndTags
     | None
 
 
@@ -34,8 +34,8 @@ isEdit m =
 
 
 type Msg
-    = Select Source
+    = Select SourceAndTags
-    | Show Source
+    | Show SourceAndTags
 
 
 update : Flags -> Msg -> ( Cmd Msg, SelectMode )
@@ -48,7 +48,7 @@ update _ msg =
             ( Cmd.none, Display source )
 
 
-view : List Source -> Html Msg
+view : List SourceAndTags -> Html Msg
 view sources =
     table [ class "ui table" ]
         [ thead []
@@ -66,7 +66,7 @@ view sources =
         ]
 
 
-renderSourceLine : Source -> Html Msg
+renderSourceLine : SourceAndTags -> Html Msg
 renderSourceLine source =
     tr
         []
@@ -82,10 +82,10 @@ renderSourceLine source =
             , a
                 [ classList
                     [ ( "ui basic tiny primary button", True )
-                    , ( "disabled", not source.enabled )
+                    , ( "disabled", not source.source.enabled )
                     ]
                 , href "#"
-                , disabled (not source.enabled)
+                , disabled (not source.source.enabled)
                 , onClick (Show source)
                 ]
                 [ i [ class "eye icon" ] []
@@ -93,25 +93,25 @@ renderSourceLine source =
                 ]
             ]
         , td [ class "collapsing" ]
-            [ text source.abbrev
+            [ text source.source.abbrev
             ]
         , td [ class "collapsing" ]
-            [ if source.enabled then
+            [ if source.source.enabled then
                 i [ class "check square outline icon" ] []
 
               else
                 i [ class "minus square outline icon" ] []
             ]
         , td [ class "collapsing" ]
-            [ source.counter |> String.fromInt |> text
+            [ source.source.counter |> String.fromInt |> text
             ]
         , td [ class "collapsing" ]
-            [ Data.Priority.fromString source.priority
+            [ Data.Priority.fromString source.source.priority
                 |> Maybe.map Data.Priority.toName
-                |> Maybe.withDefault source.priority
+                |> Maybe.withDefault source.source.priority
                 |> text
             ]
         , td []
-            [ text source.id
+            [ text source.source.id
             ]
         ]