From 855d4eefa8663a4e6ccce1286ca8d44c5690e408 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 15:33:58 +0200
Subject: [PATCH 01/15] Set progress in a linear way between each step

---
 .../src/main/scala/docspell/joex/process/ProcessItem.scala  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala
index 66d1fafa..b667d894 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala
@@ -12,11 +12,13 @@ object ProcessItem {
       cfg: Config
   )(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
     ExtractArchive(item)
+      .flatMap(Task.setProgress(20))
       .flatMap(ConvertPdf(cfg.convert, _))
+      .flatMap(Task.setProgress(40))
       .flatMap(TextExtraction(cfg.extraction, _))
-      .flatMap(Task.setProgress(50))
+      .flatMap(Task.setProgress(60))
       .flatMap(analysisOnly[F](cfg.textAnalysis))
-      .flatMap(Task.setProgress(75))
+      .flatMap(Task.setProgress(80))
       .flatMap(LinkProposal[F])
       .flatMap(Task.setProgress(99))
 

From 25d089da6c02762afeaf47ce825513795b5e6316 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 15:46:24 +0200
Subject: [PATCH 02/15] Update state and proposals only on invalid items

Invalid items are those that are not ready, and not shown to the user.
When changing metadata, it should only be changed, if the item was not
already shown to the user.
---
 .../scala/docspell/common/ItemState.scala     | 14 ++++++--
 .../joex/notify/NotifyDueItemsTask.scala      |  2 +-
 .../docspell/joex/process/ItemHandler.scala   |  6 +++-
 .../docspell/joex/process/LinkProposal.scala  | 36 +++++++++++--------
 .../restserver/conv/Conversions.scala         |  2 +-
 .../scala/docspell/store/records/RItem.scala  | 11 ++++--
 6 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/modules/common/src/main/scala/docspell/common/ItemState.scala b/modules/common/src/main/scala/docspell/common/ItemState.scala
index 506e803d..77766295 100644
--- a/modules/common/src/main/scala/docspell/common/ItemState.scala
+++ b/modules/common/src/main/scala/docspell/common/ItemState.scala
@@ -1,11 +1,18 @@
 package docspell.common
 
 import io.circe.{Decoder, Encoder}
+import cats.data.NonEmptyList
 
 sealed trait ItemState { self: Product =>
 
   final def name: String =
     productPrefix.toLowerCase
+
+  def isValid: Boolean =
+    ItemState.validStates.exists(_ == this)
+
+  def isInvalid: Boolean =
+    ItemState.invalidStates.exists(_ == this)
 }
 
 object ItemState {
@@ -24,8 +31,11 @@ object ItemState {
       case _            => Left(s"Invalid item state: $str")
     }
 
-  val validStates: Seq[ItemState] =
-    Seq(Created, Confirmed)
+  val validStates: NonEmptyList[ItemState] =
+    NonEmptyList.of(Created, Confirmed)
+
+  val invalidStates: NonEmptyList[ItemState] =
+    NonEmptyList.of(Premature, Processing)
 
   def unsafe(str: String): ItemState =
     fromString(str).fold(sys.error, identity)
diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
index 12c0d436..2b789ff1 100644
--- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala
@@ -71,7 +71,7 @@ object NotifyDueItemsTask {
         QItem.Query
           .empty(ctx.args.account.collective)
           .copy(
-            states = ItemState.validStates,
+            states = ItemState.validStates.toList,
             tagsInclude = ctx.args.tagsInclude,
             tagsExclude = ctx.args.tagsExclude,
             dueDateFrom = ctx.args.daysBack.map(back => now - Duration.days(back.toLong)),
diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
index 0ce3f9b5..4334ece4 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
@@ -25,7 +25,11 @@ object ItemHandler {
   def itemStateTask[F[_]: Sync, A](
       state: ItemState
   )(data: ItemData): Task[F, A, ItemData] =
-    Task(ctx => ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data))
+    Task(ctx =>
+      ctx.store
+        .transact(RItem.updateState(data.item.id, state, ItemState.invalidStates))
+        .map(_ => data)
+    )
 
   def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
     for {
diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala
index 7552b8db..a6d7bc16 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala
@@ -9,21 +9,26 @@ import docspell.store.records.RItem
 object LinkProposal {
 
   def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] =
-    Task { ctx =>
-      // sort by weight; order of equal weights is not important, just
-      // choose one others are then suggestions
-      // doc-date is only set when given explicitely, not from "guessing"
-      val proposals = MetaProposalList
-        .flatten(data.metas.map(_.proposals))
-        .filter(_.proposalType != MetaProposalType.DocDate)
-        .sortByWeights
+    if (data.item.state.isValid)
+      Task
+        .log[F, ProcessItemArgs](_.debug(s"Not linking proposals on existing item"))
+        .map(_ => data)
+    else
+      Task { ctx =>
+        // sort by weight; order of equal weights is not important, just
+        // choose one others are then suggestions
+        // doc-date is only set when given explicitely, not from "guessing"
+        val proposals = MetaProposalList
+          .flatten(data.metas.map(_.proposals))
+          .filter(_.proposalType != MetaProposalType.DocDate)
+          .sortByWeights
 
-      ctx.logger.info(s"Starting linking proposals") *>
-        MetaProposalType.all
-          .traverse(applyValue(data, proposals, ctx))
-          .map(result => ctx.logger.info(s"Results from proposal processing: $result"))
-          .map(_ => data)
-    }
+        ctx.logger.info(s"Starting linking proposals") *>
+          MetaProposalType.all
+            .traverse(applyValue(data, proposals, ctx))
+            .map(result => ctx.logger.info(s"Results from proposal processing: $result"))
+            .map(_ => data)
+      }
 
   def applyValue[F[_]: Sync](
       data: ItemData,
@@ -40,8 +45,9 @@ object LinkProposal {
             Result.single(mpt)
           )
       case Some(a) =>
+        val ids = a.values.map(_.ref.id.id)
         ctx.logger.info(
-          s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first."
+          s"Found many (${a.size}, ${ids}) candidates for ${a.proposalType}. Setting first."
         ) *>
           setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
             Result.multiple(mpt)
diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
index ea262bce..125d6ae0 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@@ -109,7 +109,7 @@ trait Conversions {
       coll,
       m.name,
       if (m.inbox) Seq(ItemState.Created)
-      else ItemState.validStates,
+      else ItemState.validStates.toList,
       m.direction,
       m.corrPerson,
       m.corrOrg,
diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala
index 756ef292..05f06e9f 100644
--- a/modules/store/src/main/scala/docspell/store/records/RItem.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala
@@ -1,7 +1,8 @@
 package docspell.store.records
 
-import cats.implicits._
+import cats.data.NonEmptyList
 import cats.effect.Sync
+import cats.implicits._
 import doobie._
 import doobie.implicits._
 import docspell.common._
@@ -110,12 +111,16 @@ object RItem {
   def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] =
     selectSimple(List(cid), table, id.is(itemId)).query[Ident].option
 
-  def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] =
+  def updateState(
+      itemId: Ident,
+      itemState: ItemState,
+      existing: NonEmptyList[ItemState]
+  ): ConnectionIO[Int] =
     for {
       t <- currentTime
       n <- updateRow(
         table,
-        id.is(itemId),
+        and(id.is(itemId), state.isIn(existing)),
         commas(state.setTo(itemState), updated.setTo(t))
       ).update.run
     } yield n

From f4949446e3aaa4bfea8db05065c7147d87f3a75a Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 17:16:03 +0200
Subject: [PATCH 03/15] Allow to specify an item id to amend files to existing
 items

---
 .../scala/docspell/backend/ops/OUpload.scala  | 35 ++++----
 .../docspell/common/ProcessItemArgs.scala     |  9 ++
 .../scala/docspell/joex/JoexAppImpl.scala     |  2 +-
 .../docspell/joex/process/CreateItem.scala    | 87 +++++++++++++------
 .../joex/process/ExtractArchive.scala         |  6 +-
 .../docspell/joex/process/ItemHandler.scala   | 27 +++---
 .../joex/scanmailbox/ScanMailboxTask.scala    |  2 +-
 .../docspell/joex/scheduler/Context.scala     | 10 ++-
 .../routes/IntegrationEndpointRoutes.scala    |  2 +-
 .../restserver/routes/UploadRoutes.scala      |  4 +-
 .../docspell/store/records/RAttachment.scala  |  3 +
 .../scala/docspell/store/records/RItem.scala  |  3 +
 12 files changed, 128 insertions(+), 62 deletions(-)

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
index bc2c688e..f787a17a 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -1,8 +1,9 @@
 package docspell.backend.ops
 
 import bitpeace.MimetypeHint
-import cats.implicits._
+import cats.data.OptionT
 import cats.effect._
+import cats.implicits._
 import docspell.backend.Config
 import fs2.Stream
 import docspell.common._
@@ -17,13 +18,15 @@ trait OUpload[F[_]] {
   def submit(
       data: OUpload.UploadData[F],
       account: AccountId,
-      notifyJoex: Boolean
+      notifyJoex: Boolean,
+      itemId: Option[Ident]
   ): F[OUpload.UploadResult]
 
   def submit(
       data: OUpload.UploadData[F],
       sourceId: Ident,
-      notifyJoex: Boolean
+      notifyJoex: Boolean,
+      itemId: Option[Ident]
   ): F[OUpload.UploadResult]
 }
 
@@ -68,7 +71,8 @@ object OUpload {
       def submit(
           data: OUpload.UploadData[F],
           account: AccountId,
-          notifyJoex: Boolean
+          notifyJoex: Boolean,
+          itemId: Option[Ident]
       ): F[OUpload.UploadResult] =
         for {
           files <- data.files.traverse(saveFile).map(_.flatten)
@@ -76,6 +80,7 @@ object OUpload {
           lang  <- store.transact(RCollective.findLanguage(account.collective))
           meta = ProcessItemArgs.ProcessMeta(
             account.collective,
+            itemId,
             lang.getOrElse(Language.German),
             data.meta.direction,
             data.meta.sourceAbbrev,
@@ -95,18 +100,18 @@ object OUpload {
       def submit(
           data: OUpload.UploadData[F],
           sourceId: Ident,
-          notifyJoex: Boolean
+          notifyJoex: Boolean,
+          itemId: Option[Ident]
       ): F[OUpload.UploadResult] =
-        for {
-          sOpt <-
-            store
-              .transact(RSource.find(sourceId))
-              .map(_.toRight(UploadResult.NoSource))
-          abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
-          updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
-          accId  = sOpt.map(source => AccountId(source.cid, source.sid))
-          result <- accId.traverse(acc => submit(updata, acc, notifyJoex))
-        } yield result.fold(identity, identity)
+        (for {
+          src <- OptionT(store.transact(RSource.find(sourceId)))
+          updata = data.copy(
+            meta = data.meta.copy(sourceAbbrev = src.abbrev),
+            priority = src.priority
+          )
+          accId = AccountId(src.cid, src.sid)
+          result <- OptionT.liftF(submit(updata, accId, notifyJoex, itemId))
+        } yield result).getOrElse(UploadResult.NoSource)
 
       private def submitJobs(
           notifyJoex: Boolean
diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala
index 73e7a951..d170eae0 100644
--- a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala
+++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala
@@ -4,6 +4,14 @@ import io.circe._, io.circe.generic.semiauto._
 import docspell.common.syntax.all._
 import ProcessItemArgs._
 
+/** Arguments to the process-item task.
+  *
+  * This task is run for each new file to create a new item from it or
+  * to add this file as an attachment to an existing item.
+  *
+  * If the `itemId' is set to some value, the item is tried to load to
+  * ammend with the given files. Otherwise a new item is created.
+  */
 case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
 
   def makeSubject: String =
@@ -22,6 +30,7 @@ object ProcessItemArgs {
 
   case class ProcessMeta(
       collective: Ident,
+      itemId: Option[Ident],
       language: Language,
       direction: Option[Direction],
       sourceAbbrev: String,
diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
index d07ca841..f2d3cd91 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
@@ -77,7 +77,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             ProcessItemArgs.taskName,
-            ItemHandler[F](cfg),
+            ItemHandler.newItem[F](cfg),
             ItemHandler.onCancel[F]
           )
         )
diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
index 595c0b1b..30737bba 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
@@ -32,44 +32,75 @@ object CreateItem {
 
       def fileMetas(itemId: Ident, now: Timestamp) =
         Stream
-          .emits(ctx.args.files)
-          .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
-          .collect({ case (f, Some(fm)) if isValidFile(fm) => f })
-          .zipWithIndex
-          .evalMap({
-            case (f, index) =>
-              Ident
-                .randomId[F]
-                .map(id =>
-                  RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)
-                )
-          })
+          .eval(ctx.store.transact(RAttachment.countOnItem(itemId)))
+          .flatMap { offset =>
+            Stream
+              .emits(ctx.args.files)
+              .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
+              .collect({ case (f, Some(fm)) if isValidFile(fm) => f })
+              .zipWithIndex
+              .evalMap({
+                case (f, index) =>
+                  Ident
+                    .randomId[F]
+                    .map(id =>
+                      RAttachment(
+                        id,
+                        itemId,
+                        f.fileMetaId,
+                        index.toInt + offset,
+                        now,
+                        f.name
+                      )
+                    )
+              })
+          }
           .compile
           .toVector
 
-      val item = RItem.newItem[F](
-        ctx.args.meta.collective,
-        ctx.args.makeSubject,
-        ctx.args.meta.sourceAbbrev,
-        ctx.args.meta.direction.getOrElse(Direction.Incoming),
-        ItemState.Premature
-      )
+      val loadItemOrInsertNew =
+        ctx.args.meta.itemId match {
+          case Some(id) =>
+            (for {
+              _ <- OptionT.liftF(
+                ctx.logger.info(
+                  s"Loading item with id ${id.id} to ammend"
+                )
+              )
+              item <- OptionT(
+                ctx.store
+                  .transact(RItem.findByIdAndCollective(id, ctx.args.meta.collective))
+              )
+            } yield (1, item))
+              .getOrElseF(Sync[F].raiseError(new Exception(s"Item not found.")))
+          case None =>
+            for {
+              _ <- ctx.logger.info(
+                s"Creating new item with ${ctx.args.files.size} attachment(s)"
+              )
+              item <- RItem.newItem[F](
+                ctx.args.meta.collective,
+                ctx.args.makeSubject,
+                ctx.args.meta.sourceAbbrev,
+                ctx.args.meta.direction.getOrElse(Direction.Incoming),
+                ItemState.Premature
+              )
+              n <- ctx.store.transact(RItem.insert(item))
+            } yield (n, item)
+        }
 
       for {
-        _ <- ctx.logger.info(
-          s"Creating new item with ${ctx.args.files.size} attachment(s)"
-        )
         time <- Duration.stopTime[F]
-        it   <- item
-        n    <- ctx.store.transact(RItem.insert(it))
-        _    <- if (n != 1) storeItemError[F](ctx) else ().pure[F]
-        fm   <- fileMetas(it.id, it.created)
+        it   <- loadItemOrInsertNew
+        _    <- if (it._1 != 1) storeItemError[F](ctx) else ().pure[F]
+        now  <- Timestamp.current[F]
+        fm   <- fileMetas(it._2.id, now)
         k    <- fm.traverse(insertAttachment(ctx))
         _    <- logDifferences(ctx, fm, k.sum)
         dur  <- time
         _    <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}")
       } yield ItemData(
-        it,
+        it._2,
         fm,
         Vector.empty,
         Vector.empty,
@@ -86,7 +117,7 @@ object CreateItem {
     } yield n)
   }
 
-  def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
+  private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
     Task { ctx =>
       for {
         cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
index ddb184ab..d3a156ff 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
@@ -50,8 +50,10 @@ object ExtractArchive {
         findMime(ctx)(ra).flatMap(m => extractSafe(ctx, archive)(ra, m))
 
       for {
-        ras <- item.attachments.traverse(extract)
-        nra = ras.flatMap(_.files).zipWithIndex.map(t => t._1.copy(position = t._2))
+        ras     <- item.attachments.traverse(extract)
+        lastPos <- ctx.store.transact(RAttachment.countOnItem(item.item.id))
+        nra =
+          ras.flatMap(_.files).zipWithIndex.map(t => t._1.copy(position = lastPos + t._2))
         _ <- nra.traverse(storeAttachment(ctx))
         naa = ras.flatMap(_.archives)
         _ <- naa.traverse(storeArchive(ctx))
diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
index 4334ece4..dbc0f70a 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
@@ -2,19 +2,20 @@ package docspell.joex.process
 
 import cats.implicits._
 import cats.effect._
+import fs2.Stream
 import docspell.common.{ItemState, ProcessItemArgs}
 import docspell.joex.Config
-import docspell.joex.scheduler.{Context, Task}
+import docspell.joex.scheduler.Task
 import docspell.store.queries.QItem
-import docspell.store.records.{RItem, RJob}
+import docspell.store.records.RItem
 
 object ItemHandler {
   def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] =
     logWarn("Now cancelling. Deleting potentially created data.").flatMap(_ =>
-      deleteByFileIds
+      deleteByFileIds.flatMap(_ => deleteFiles)
     )
 
-  def apply[F[_]: ConcurrentEffect: ContextShift](
+  def newItem[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config
   ): Task[F, ProcessItemArgs, Unit] =
     CreateItem[F]
@@ -31,16 +32,13 @@ object ItemHandler {
         .map(_ => data)
     )
 
-  def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
-    for {
-      current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
-      last = ctx.config.retries == current.getOrElse(0)
-    } yield last
+  def isLastRetry[F[_]: Sync]: Task[F, ProcessItemArgs, Boolean] =
+    Task(_.isLastRetry)
 
   def safeProcess[F[_]: ConcurrentEffect: ContextShift](
       cfg: Config
   )(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
-    Task(isLastRetry[F, ProcessItemArgs] _).flatMap {
+    isLastRetry[F].flatMap {
       case true =>
         ProcessItem[F](cfg)(data).attempt.flatMap({
           case Right(d) =>
@@ -64,6 +62,15 @@ object ItemHandler {
       } yield ()
     }
 
+  private def deleteFiles[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
+    Task(ctx =>
+      Stream
+        .emits(ctx.args.files.map(_.fileMetaId.id))
+        .flatMap(id => ctx.store.bitpeace.delete(id).attempt.drain)
+        .compile
+        .drain
+    )
+
   private def logWarn[F[_]](msg: => String): Task[F, ProcessItemArgs, Unit] =
     Task(_.logger.warn(msg))
 }
diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index 670b2fec..8d59b481 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -259,7 +259,7 @@ object ScanMailboxTask {
           priority = Priority.Low,
           tracker = None
         )
-        res <- upload.submit(data, ctx.args.account, false)
+        res <- upload.submit(data, ctx.args.account, false, None)
       } yield res
     }
 
diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala
index ca16c1a8..d8a98906 100644
--- a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala
@@ -1,7 +1,7 @@
 package docspell.joex.scheduler
 
-import cats.Functor
-import cats.effect.{Blocker, Concurrent}
+import cats.{Applicative, Functor}
+import cats.effect._
 import cats.implicits._
 import docspell.common._
 import docspell.store.Store
@@ -23,6 +23,12 @@ trait Context[F[_], A] { self =>
 
   def store: Store[F]
 
+  final def isLastRetry(implicit ev: Applicative[F]): F[Boolean] =
+    for {
+      current <- store.transact(RJob.getRetries(jobId))
+      last = config.retries == current.getOrElse(0)
+    } yield last
+
   def blocker: Blocker
 
   def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] =
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/IntegrationEndpointRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/IntegrationEndpointRoutes.scala
index 6f0361d4..9433fa01 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/IntegrationEndpointRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/IntegrationEndpointRoutes.scala
@@ -80,7 +80,7 @@ object IntegrationEndpointRoutes {
         cfg.backend.files.validMimeTypes
       )
       account = AccountId(coll, Ident.unsafe("docspell-system"))
-      result <- backend.upload.submit(updata, account, true)
+      result <- backend.upload.submit(updata, account, true, None)
       res    <- Ok(basicResult(result))
     } yield res
   }
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
index d48d5159..77dfb427 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
@@ -36,7 +36,7 @@ object UploadRoutes {
             Priority.High,
             cfg.backend.files.validMimeTypes
           )
-          result <- backend.upload.submit(updata, user.account, true)
+          result <- backend.upload.submit(updata, user.account, true, None)
           res    <- Ok(basicResult(result))
         } yield res
     }
@@ -56,7 +56,7 @@ object UploadRoutes {
             Priority.Low,
             cfg.backend.files.validMimeTypes
           )
-          result <- backend.upload.submit(updata, id, true)
+          result <- backend.upload.submit(updata, id, true, None)
           res    <- Ok(basicResult(result))
         } yield res
     }
diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
index a63dd4d5..dbb5dc16 100644
--- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
@@ -38,6 +38,9 @@ object RAttachment {
       fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}"
     ).update.run
 
+  def countOnItem(id: Ident): ConnectionIO[Int] =
+    selectCount(itemId, table, itemId.is(id)).query[Int].unique
+
   def updateFileIdAndName(
       attachId: Ident,
       fId: Ident,
diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala
index 05f06e9f..6d300e62 100644
--- a/modules/store/src/main/scala/docspell/store/records/RItem.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala
@@ -290,4 +290,7 @@ object RItem {
 
   def existsById(itemId: Ident): ConnectionIO[Boolean] =
     selectCount(id, table, id.is(itemId)).query[Int].unique.map(_ > 0)
+
+  def findByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Option[RItem]] =
+    selectSimple(all, table, and(id.is(itemId), cid.is(coll))).query[RItem].option
 }

From a5ca3b03259de3cd61d20996b99d7d4cc940bc5b Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 19:42:36 +0200
Subject: [PATCH 04/15] Add routes and upload form to item detail

---
 .../src/main/resources/docspell-openapi.yml   | 105 ++++++-
 .../restserver/routes/UploadRoutes.scala      |  26 ++
 modules/webapp/src/main/elm/Api.elm           |  47 ++-
 .../webapp/src/main/elm/Comp/ItemDetail.elm   | 275 +++++++++++++++++-
 modules/webapp/src/main/elm/Data/Icons.elm    |  14 +-
 .../src/main/elm/Page/Upload/Update.elm       |   7 +-
 6 files changed, 456 insertions(+), 18 deletions(-)

diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml
index e0ecf20f..5932a998 100644
--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -87,12 +87,6 @@ paths:
         The upload meta data can be used to tell, whether multiple
         files are one item, or if each file should become a single
         item. By default, each file will be a one item.
-
-        Only certain file types are supported:
-
-        * application/pdf
-
-        Support for more types might be added.
       parameters:
         - $ref: "#/components/parameters/id"
       requestBody:
@@ -115,6 +109,48 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/BasicResult"
+  /open/upload/item/{itemId}/{id}:
+    post:
+      tags: [ Upload ]
+      summary: Upload files to docspell.
+      description: |
+        Upload a file to docspell for processing. The id is a *source
+        id* configured by a collective. Files are submitted for
+        processing which eventually resuts in an item in the inbox of
+        the corresponding collective. This endpoint associates the
+        files to an existing item identified by its `itemId`.
+
+        The request must be a `multipart/form-data` request, where the
+        first part has name `meta`, is optional and may contain upload
+        metadata as JSON. Checkout the structure `ItemUploadMeta` at
+        the end if it is not shown here. Other parts specify the
+        files. Multiple files can be specified, but at least on is
+        required.
+
+        Upload meta data is ignored.
+      parameters:
+        - $ref: "#/components/parameters/id"
+        - $ref: "#/components/parameters/itemId"
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                meta:
+                  $ref: "#/components/schemas/ItemUploadMeta"
+                file:
+                  type: array
+                  items:
+                    type: string
+                    format: binary
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
   /sec/checkfile/{checksum}:
     get:
       tags: [ Upload ]
@@ -155,12 +191,6 @@ paths:
         The upload meta data can be used to tell, whether multiple
         files are one item, or if each file should become a single
         item. By default, each file will be a one item.
-
-        Only certain file types are supported:
-
-        * application/pdf
-
-        Support for more types might be added.
       security:
         - authTokenHeader: []
       requestBody:
@@ -183,6 +213,50 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/BasicResult"
+  /sec/upload/{itemId}:
+    post:
+      tags: [ Upload ]
+      summary: Upload files to docspell.
+      description: |
+        Upload files to docspell for processing. This route is meant
+        for authenticated users that upload files to their account.
+        This endpoint will associate the files to an existing item
+        identified by its `itemId`.
+
+        Everything else is the same as with the
+        `/open/upload/item/{itemId}/{id}` endpoint.
+
+        The request must be a "multipart/form-data" request, where the
+        first part is optional and may contain upload metadata as
+        JSON. Other parts specify the files. Multiple files can be
+        specified, but at least on is required.
+
+        The upload meta data is ignored, since the item already
+        exists.
+      security:
+        - authTokenHeader: []
+      parameters:
+        - $ref: "#/components/parameters/itemId"
+      requestBody:
+        content:
+          multipart/form-data:
+            schema:
+              type: object
+              properties:
+                meta:
+                  $ref: "#/components/schemas/ItemUploadMeta"
+                file:
+                  type: array
+                  items:
+                    type: string
+                    format: binary
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
   /open/signup/register:
     post:
       tags: [ Registration ]
@@ -3156,6 +3230,13 @@ components:
       required: true
       schema:
         type: string
+    itemId:
+      name: itemId
+      in: path
+      description: An identifier for an item
+      required: true
+      schema:
+        type: string
     full:
       name: full
       in: query
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
index 77dfb427..afec48a4 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
@@ -39,6 +39,19 @@ object UploadRoutes {
           result <- backend.upload.submit(updata, user.account, true, None)
           res    <- Ok(basicResult(result))
         } yield res
+
+      case req @ POST -> Root / "item" / Ident(itemId) =>
+        for {
+          multipart <- req.as[Multipart[F]]
+          updata <- readMultipart(
+            multipart,
+            logger,
+            Priority.High,
+            cfg.backend.files.validMimeTypes
+          )
+          result <- backend.upload.submit(updata, user.account, true, Some(itemId))
+          res    <- Ok(basicResult(result))
+        } yield res
     }
   }
 
@@ -59,6 +72,19 @@ object UploadRoutes {
           result <- backend.upload.submit(updata, id, true, None)
           res    <- Ok(basicResult(result))
         } yield res
+
+      case req @ POST -> Root / "item" / Ident(itemId) / Ident(id) =>
+        for {
+          multipart <- req.as[Multipart[F]]
+          updata <- readMultipart(
+            multipart,
+            logger,
+            Priority.Low,
+            cfg.backend.files.validMimeTypes
+          )
+          result <- backend.upload.submit(updata, id, true, Some(itemId))
+          res    <- Ok(basicResult(result))
+        } yield res
     }
   }
 }
diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm
index 37b2d4f4..4a085d4f 100644
--- a/modules/webapp/src/main/elm/Api.elm
+++ b/modules/webapp/src/main/elm/Api.elm
@@ -71,6 +71,7 @@ module Api exposing
     , submitNotifyDueItems
     , updateScanMailbox
     , upload
+    , uploadAmend
     , uploadSingle
     , versionInfo
     )
@@ -429,7 +430,42 @@ createImapSettings flags mname ems receive =
 --- Upload
 
 
-upload : Flags -> Maybe String -> ItemUploadMeta -> List File -> (String -> Result Http.Error BasicResult -> msg) -> List (Cmd msg)
+uploadAmend :
+    Flags
+    -> String
+    -> List File
+    -> (String -> Result Http.Error BasicResult -> msg)
+    -> List (Cmd msg)
+uploadAmend flags itemId files receive =
+    let
+        mkReq file =
+            let
+                fid =
+                    Util.File.makeFileId file
+
+                path =
+                    "/api/v1/sec/upload/item/" ++ itemId
+            in
+            Http2.authPostTrack
+                { url = flags.config.baseUrl ++ path
+                , account = getAccount flags
+                , body =
+                    Http.multipartBody <|
+                        [ Http.filePart "file[]" file ]
+                , expect = Http.expectJson (receive fid) Api.Model.BasicResult.decoder
+                , tracker = fid
+                }
+    in
+    List.map mkReq files
+
+
+upload :
+    Flags
+    -> Maybe String
+    -> ItemUploadMeta
+    -> List File
+    -> (String -> Result Http.Error BasicResult -> msg)
+    -> List (Cmd msg)
 upload flags sourceId meta files receive =
     let
         metaStr =
@@ -457,7 +493,14 @@ upload flags sourceId meta files receive =
     List.map mkReq files
 
 
-uploadSingle : Flags -> Maybe String -> ItemUploadMeta -> String -> List File -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+uploadSingle :
+    Flags
+    -> Maybe String
+    -> ItemUploadMeta
+    -> String
+    -> List File
+    -> (Result Http.Error BasicResult -> msg)
+    -> Cmd msg
 uploadSingle flags sourceId meta track files receive =
     let
         metaStr =
diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
index 6ef0569a..3f24726b 100644
--- a/modules/webapp/src/main/elm/Comp/ItemDetail.elm
+++ b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
@@ -25,6 +25,7 @@ import Browser.Navigation as Nav
 import Comp.AttachmentMeta
 import Comp.DatePicker
 import Comp.Dropdown exposing (isDropdownChangeMsg)
+import Comp.Dropzone
 import Comp.ItemMail
 import Comp.MarkdownInput
 import Comp.SentMails
@@ -34,12 +35,16 @@ import Data.Flags exposing (Flags)
 import Data.Icons as Icons
 import DatePicker exposing (DatePicker)
 import Dict exposing (Dict)
+import File exposing (File)
 import Html exposing (..)
 import Html.Attributes exposing (..)
 import Html.Events exposing (onCheck, onClick, onInput)
 import Http
 import Markdown
 import Page exposing (Page(..))
+import Ports
+import Set exposing (Set)
+import Util.File exposing (makeFileId)
 import Util.Http
 import Util.List
 import Util.Maybe
@@ -77,6 +82,12 @@ type alias Model =
     , attachMetaOpen : Bool
     , pdfNativeView : Bool
     , deleteAttachConfirm : Comp.YesNoDimmer.Model
+    , addFilesOpen : Bool
+    , addFilesModel : Comp.Dropzone.Model
+    , selectedFiles : List File
+    , completed : Set String
+    , errored : Set String
+    , loading : Set String
     }
 
 
@@ -165,6 +176,12 @@ emptyModel =
     , attachMetaOpen = False
     , pdfNativeView = False
     , deleteAttachConfirm = Comp.YesNoDimmer.emptyModel
+    , addFilesOpen = False
+    , addFilesModel = Comp.Dropzone.init Comp.Dropzone.defaultSettings
+    , selectedFiles = []
+    , completed = Set.empty
+    , errored = Set.empty
+    , loading = Set.empty
     }
 
 
@@ -221,6 +238,12 @@ type Msg
     | RequestDeleteAttachment String
     | DeleteAttachConfirm String Comp.YesNoDimmer.Msg
     | DeleteAttachResp (Result Http.Error BasicResult)
+    | AddFilesToggle
+    | AddFilesMsg Comp.Dropzone.Msg
+    | AddFilesSubmitUpload
+    | AddFilesUploadResp String (Result Http.Error BasicResult)
+    | AddFilesProgress String Http.Progress
+    | AddFilesReset
 
 
 
@@ -334,6 +357,42 @@ setDueDate flags model date =
     Api.setItemDueDate flags model.item.id (OptionalDate date) SaveResp
 
 
+isLoading : Model -> File -> Bool
+isLoading model file =
+    Set.member (makeFileId file) model.loading
+
+
+isCompleted : Model -> File -> Bool
+isCompleted model file =
+    Set.member (makeFileId file) model.completed
+
+
+isError : Model -> File -> Bool
+isError model file =
+    Set.member (makeFileId file) model.errored
+
+
+isIdle : Model -> File -> Bool
+isIdle model file =
+    not (isLoading model file || isCompleted model file || isError model file)
+
+
+setCompleted : Model -> String -> Set String
+setCompleted model fileid =
+    Set.insert fileid model.completed
+
+
+setErrored : Model -> String -> Set String
+setErrored model fileid =
+    Set.insert fileid model.errored
+
+
+isSuccessAll : Model -> Bool
+isSuccessAll model =
+    List.map makeFileId model.selectedFiles
+        |> List.all (\id -> Set.member id model.completed)
+
+
 update : Nav.Key -> Flags -> Maybe String -> Msg -> Model -> ( Model, Cmd Msg )
 update key flags next msg model =
     case msg of
@@ -430,6 +489,9 @@ update key flags next msg model =
                         )
                         m5
 
+                ( m7, c7 ) =
+                    update key flags next AddFilesReset m6
+
                 proposalCmd =
                     if item.state == "created" then
                         Api.getItemProposals flags item.id GetProposalResp
@@ -437,7 +499,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( { m6
+            ( { m7
                 | item = item
                 , nameModel = item.name
                 , notesModel = item.notes
@@ -453,6 +515,7 @@ update key flags next msg model =
                 , c4
                 , c5
                 , c6
+                , c7
                 , getOptions flags
                 , proposalCmd
                 , Api.getSentMails flags item.id SentMailsResp
@@ -980,6 +1043,107 @@ update key flags next msg model =
                 (DeleteAttachConfirm id Comp.YesNoDimmer.activate)
                 model
 
+        AddFilesToggle ->
+            ( { model | addFilesOpen = not model.addFilesOpen }
+            , Cmd.none
+            )
+
+        AddFilesMsg lm ->
+            let
+                ( dm, dc, df ) =
+                    Comp.Dropzone.update lm model.addFilesModel
+
+                nextFiles =
+                    model.selectedFiles ++ df
+            in
+            ( { model | addFilesModel = dm, selectedFiles = nextFiles }
+            , Cmd.map AddFilesMsg dc
+            )
+
+        AddFilesReset ->
+            ( { model
+                | selectedFiles = []
+                , addFilesModel = Comp.Dropzone.init Comp.Dropzone.defaultSettings
+                , completed = Set.empty
+                , errored = Set.empty
+                , loading = Set.empty
+              }
+            , Cmd.none
+            )
+
+        AddFilesSubmitUpload ->
+            let
+                fileids =
+                    List.map makeFileId model.selectedFiles
+
+                uploads =
+                    Cmd.batch (Api.uploadAmend flags model.item.id model.selectedFiles AddFilesUploadResp)
+
+                tracker =
+                    Sub.batch <| List.map (\id -> Http.track id (AddFilesProgress id)) fileids
+
+                ( cm2, _, _ ) =
+                    Comp.Dropzone.update (Comp.Dropzone.setActive False) model.addFilesModel
+            in
+            ( { model | loading = Set.fromList fileids, addFilesModel = cm2 }, uploads )
+
+        AddFilesUploadResp fileid (Ok res) ->
+            let
+                compl =
+                    if res.success then
+                        setCompleted model fileid
+
+                    else
+                        model.completed
+
+                errs =
+                    if not res.success then
+                        setErrored model fileid
+
+                    else
+                        model.errored
+
+                load =
+                    Set.remove fileid model.loading
+
+                newModel =
+                    { model | completed = compl, errored = errs, loading = load }
+            in
+            ( newModel
+            , Ports.setProgress ( fileid, 100 )
+            )
+
+        AddFilesUploadResp fileid (Err _) ->
+            let
+                errs =
+                    setErrored model fileid
+
+                load =
+                    Set.remove fileid model.loading
+            in
+            ( { model | errored = errs, loading = load }, Cmd.none )
+
+        AddFilesProgress fileid progress ->
+            let
+                percent =
+                    case progress of
+                        Http.Sending p ->
+                            Http.fractionSent p
+                                |> (*) 100
+                                |> round
+
+                        _ ->
+                            0
+
+                updateBars =
+                    if percent == 0 then
+                        Cmd.none
+
+                    else
+                        Ports.setProgress ( fileid, percent )
+            in
+            ( model, updateBars )
+
 
 
 -- view
@@ -1001,7 +1165,7 @@ view inav model =
         , div
             [ classList
                 [ ( "ui ablue-comp menu", True )
-                , ( "top attached", model.mailOpen )
+                , ( "top attached", model.mailOpen || model.addFilesOpen )
                 ]
             ]
             [ a [ class "item", Page.href HomePage ]
@@ -1066,8 +1230,24 @@ view inav model =
                 ]
                 [ Icons.editNotesIcon
                 ]
+            , a
+                [ classList
+                    [ ( "toggle item", True )
+                    , ( "active", model.addFilesOpen )
+                    ]
+                , if model.addFilesOpen then
+                    title "Close"
+
+                  else
+                    title "Add Files"
+                , onClick AddFilesToggle
+                , href "#"
+                ]
+                [ Icons.addFilesIcon
+                ]
             ]
         , renderMailForm model
+        , renderAddFilesForm model
         , div [ class "ui grid" ]
             [ Html.map DeleteItemConfirm (Comp.YesNoDimmer.view model.deleteItemConfirm)
             , div
@@ -1756,3 +1936,94 @@ renderMailForm model =
                 |> text
             ]
         ]
+
+
+renderAddFilesForm : Model -> Html Msg
+renderAddFilesForm model =
+    div
+        [ classList
+            [ ( "ui bottom attached segment", True )
+            , ( "invisible hidden", not model.addFilesOpen )
+            ]
+        ]
+        [ h4 [ class "ui header" ]
+            [ text "Add more files to this item"
+            ]
+        , Html.map AddFilesMsg (Comp.Dropzone.view model.addFilesModel)
+        , button
+            [ class "ui primary button"
+            , href "#"
+            , onClick AddFilesSubmitUpload
+            ]
+            [ text "Submit"
+            ]
+        , button
+            [ class "ui secondary button"
+            , href "#"
+            , onClick AddFilesReset
+            ]
+            [ text "Reset"
+            ]
+        , div
+            [ classList
+                [ ( "ui success message", True )
+                , ( "invisible hidden", model.selectedFiles == [] || not (isSuccessAll model) )
+                ]
+            ]
+            [ text "All files have been uploaded. They are being processed, some data "
+            , text "may not be available immediately. "
+            , a
+                [ class "link"
+                , href "#"
+                , onClick ReloadItem
+                ]
+                [ text "Refresh now"
+                ]
+            ]
+        , div [ class "ui items" ]
+            (List.map (renderFileItem model) model.selectedFiles)
+        ]
+
+
+renderFileItem : Model -> File -> Html Msg
+renderFileItem model file =
+    let
+        name =
+            File.name file
+
+        size =
+            File.size file
+                |> toFloat
+                |> Util.Size.bytesReadable Util.Size.B
+    in
+    div [ class "item" ]
+        [ i
+            [ classList
+                [ ( "large", True )
+                , ( "file outline icon", isIdle model file )
+                , ( "loading spinner icon", isLoading model file )
+                , ( "green check icon", isCompleted model file )
+                , ( "red bolt icon", isError model file )
+                ]
+            ]
+            []
+        , div [ class "middle aligned content" ]
+            [ div [ class "header" ]
+                [ text name
+                ]
+            , div [ class "right floated meta" ]
+                [ text size
+                ]
+            , div [ class "description" ]
+                [ div
+                    [ classList
+                        [ ( "ui small indicating progress", True )
+                        ]
+                    , id (makeFileId file)
+                    ]
+                    [ div [ class "bar" ]
+                        []
+                    ]
+                ]
+            ]
+        ]
diff --git a/modules/webapp/src/main/elm/Data/Icons.elm b/modules/webapp/src/main/elm/Data/Icons.elm
index ed90ebb3..13d63503 100644
--- a/modules/webapp/src/main/elm/Data/Icons.elm
+++ b/modules/webapp/src/main/elm/Data/Icons.elm
@@ -1,5 +1,7 @@
 module Data.Icons exposing
-    ( concerned
+    ( addFiles
+    , addFilesIcon
+    , concerned
     , concernedIcon
     , correspondent
     , correspondentIcon
@@ -51,3 +53,13 @@ editNotes =
 editNotesIcon : Html msg
 editNotesIcon =
     i [ class editNotes ] []
+
+
+addFiles : String
+addFiles =
+    "file plus icon"
+
+
+addFilesIcon : Html msg
+addFilesIcon =
+    i [ class addFiles ] []
diff --git a/modules/webapp/src/main/elm/Page/Upload/Update.elm b/modules/webapp/src/main/elm/Page/Upload/Update.elm
index b12e8904..9f8eb06b 100644
--- a/modules/webapp/src/main/elm/Page/Upload/Update.elm
+++ b/modules/webapp/src/main/elm/Page/Upload/Update.elm
@@ -41,7 +41,12 @@ update sourceId flags msg model =
 
                 uploads =
                     if model.singleItem then
-                        Api.uploadSingle flags sourceId meta uploadAllTracker model.files (SingleUploadResp uploadAllTracker)
+                        Api.uploadSingle flags
+                            sourceId
+                            meta
+                            uploadAllTracker
+                            model.files
+                            (SingleUploadResp uploadAllTracker)
 
                     else
                         Cmd.batch (Api.upload flags sourceId meta model.files SingleUploadResp)

From b8267f60c1cdc1b1541607b876025a1f3eca3ca6 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 19:50:39 +0200
Subject: [PATCH 05/15] Fix edit notes rendering

It must be outside the iframe tree, to not modify it when it
appears/disappears.
---
 .../webapp/src/main/elm/Comp/ItemDetail.elm   | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
index 3f24726b..901558f4 100644
--- a/modules/webapp/src/main/elm/Comp/ItemDetail.elm
+++ b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
@@ -1165,7 +1165,11 @@ view inav model =
         , div
             [ classList
                 [ ( "ui ablue-comp menu", True )
-                , ( "top attached", model.mailOpen || model.addFilesOpen )
+                , ( "top attached"
+                  , model.mailOpen
+                        || model.addFilesOpen
+                        || isEditNotes model.notesField
+                  )
                 ]
             ]
             [ a [ class "item", Page.href HomePage ]
@@ -1248,6 +1252,7 @@ view inav model =
             ]
         , renderMailForm model
         , renderAddFilesForm model
+        , renderNotes model
         , div [ class "ui grid" ]
             [ Html.map DeleteItemConfirm (Comp.YesNoDimmer.view model.deleteItemConfirm)
             , div
@@ -1271,8 +1276,7 @@ view inav model =
                 ]
               <|
                 List.concat
-                    [ renderNotes model
-                    , [ renderAttachmentsTabMenu model
+                    [ [ renderAttachmentsTabMenu model
                       ]
                     , renderAttachmentsTabBody model
                     , renderIdInfo model
@@ -1297,16 +1301,16 @@ renderIdInfo model =
     ]
 
 
-renderNotes : Model -> List (Html Msg)
+renderNotes : Model -> Html Msg
 renderNotes model =
     case model.notesField of
         HideNotes ->
             case model.item.notes of
                 Nothing ->
-                    []
+                    span [ class "invisible hidden" ] []
 
                 Just _ ->
-                    [ div [ class "ui segment" ]
+                    div [ class "ui segment" ]
                         [ a
                             [ class "ui top left attached label"
                             , onClick ToggleNotes
@@ -1316,15 +1320,14 @@ renderNotes model =
                             , text "Show notes…"
                             ]
                         ]
-                    ]
 
         ViewNotes ->
             case model.item.notes of
                 Nothing ->
-                    []
+                    span [ class "hidden invisible" ] []
 
                 Just str ->
-                    [ div [ class "ui segment" ]
+                    div [ class "ui segment" ]
                         [ Markdown.toHtml [ class "item-notes" ] str
                         , a
                             [ class "ui left corner label"
@@ -1334,10 +1337,9 @@ renderNotes model =
                             [ i [ class "eye slash icon" ] []
                             ]
                         ]
-                    ]
 
         EditNotes mm ->
-            [ div [ class "ui segment" ]
+            div [ class "ui bottom attached segment" ]
                 [ Html.map NotesEditMsg (Comp.MarkdownInput.view (Maybe.withDefault "" model.notesModel) mm)
                 , div [ class "ui secondary menu" ]
                     [ a
@@ -1358,7 +1360,6 @@ renderNotes model =
                         ]
                     ]
                 ]
-            ]
 
 
 attachmentVisible : Model -> Int -> Bool

From 6501060730d7b609643268a903f1a2c1e719204a Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sat, 23 May 2020 20:09:21 +0200
Subject: [PATCH 06/15] Make notes more prominent. Fix heading in mail form

---
 modules/webapp/src/main/elm/Comp/ItemDetail.elm | 10 +++++-----
 modules/webapp/src/main/webjar/docspell.css     |  3 +++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
index 901558f4..5a883321 100644
--- a/modules/webapp/src/main/elm/Comp/ItemDetail.elm
+++ b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
@@ -1327,7 +1327,7 @@ renderNotes model =
                     span [ class "hidden invisible" ] []
 
                 Just str ->
-                    div [ class "ui segment" ]
+                    div [ class "ui raised segment item-notes-display" ]
                         [ Markdown.toHtml [ class "item-notes" ] str
                         , a
                             [ class "ui left corner label"
@@ -1903,7 +1903,10 @@ renderMailForm model =
             , ( "invisible hidden", not model.mailOpen )
             ]
         ]
-        [ div
+        [ h4 [ class "ui header" ]
+            [ text "Send this item via E-Mail"
+            ]
+        , div
             [ classList
                 [ ( "ui dimmer", True )
                 , ( "active", model.mailSending )
@@ -1913,9 +1916,6 @@ renderMailForm model =
                 [ text "Sending …"
                 ]
             ]
-        , h4 [ class "ui header" ]
-            [ text "Send this item via E-Mail"
-            ]
         , Html.map ItemMailMsg (Comp.ItemMail.view model.itemMail)
         , div
             [ classList
diff --git a/modules/webapp/src/main/webjar/docspell.css b/modules/webapp/src/main/webjar/docspell.css
index 2c9c204d..ff0e3f8f 100644
--- a/modules/webapp/src/main/webjar/docspell.css
+++ b/modules/webapp/src/main/webjar/docspell.css
@@ -70,6 +70,9 @@
 .default-layout .ui.segment .item-notes {
     padding: 0 1em;
 }
+.default-layout .ui.segment.item-notes-display {
+    background: rgba(246, 255, 158, 0.4);
+}
 
 .default-layout .extracted-text {
     font-family: monospace;

From 075b665c681b7b1f1f5d035d93a050b84103f2ad Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 11:46:45 +0200
Subject: [PATCH 07/15] Add some more tlds to look for

---
 .../src/main/scala/docspell/analysis/contact/Tld.scala   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/modules/analysis/src/main/scala/docspell/analysis/contact/Tld.scala b/modules/analysis/src/main/scala/docspell/analysis/contact/Tld.scala
index 3f8ba6ae..f8095041 100644
--- a/modules/analysis/src/main/scala/docspell/analysis/contact/Tld.scala
+++ b/modules/analysis/src/main/scala/docspell/analysis/contact/Tld.scala
@@ -19,6 +19,15 @@ private[analysis] object Tld {
     ".edu",
     ".gov",
     ".mil",
+    ".info",
+    ".app",
+    ".bar",
+    ".biz",
+    ".club",
+    ".coop",
+    ".icu",
+    ".name",
+    ".xyz",
     ".ad",
     ".ae",
     ".al",

From f519a8effaa23d177e03723efa2d6997802cd6cf Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 10:58:50 +0200
Subject: [PATCH 08/15] Check for an existing item before attempting to add
 files

---
 .../scala/docspell/backend/ops/OUpload.scala  | 74 ++++++++++++++-----
 .../restserver/conv/Conversions.scala         |  1 +
 2 files changed, 58 insertions(+), 17 deletions(-)

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
index f787a17a..59b73144 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -1,7 +1,8 @@
 package docspell.backend.ops
 
 import bitpeace.MimetypeHint
-import cats.data.OptionT
+import cats.Functor
+import cats.data.{EitherT, OptionT}
 import cats.effect._
 import cats.implicits._
 import docspell.backend.Config
@@ -10,7 +11,7 @@ import docspell.common._
 import docspell.common.syntax.all._
 import docspell.store.Store
 import docspell.store.queue.JobQueue
-import docspell.store.records.{RCollective, RJob, RSource}
+import docspell.store.records._
 import org.log4s._
 
 trait OUpload[F[_]] {
@@ -55,11 +56,32 @@ object OUpload {
 
   sealed trait UploadResult
   object UploadResult {
-    case object Success  extends UploadResult
-    case object NoFiles  extends UploadResult
+
+    /** File(s) have been successfully submitted. */
+    case object Success extends UploadResult
+
+    def success: UploadResult = Success
+
+    /** There were no files to submit. */
+    case object NoFiles extends UploadResult
+
+    def noFiles: UploadResult = NoFiles
+
+    /** A source (`RSource') could not be found for a given source-id. */
     case object NoSource extends UploadResult
+
+    def noSource: UploadResult = NoSource
+
+    /** When adding files to an item, no item was found using the given
+      * item-id. */
+    case object NoItem extends UploadResult
+
+    def noItem: UploadResult = NoItem
   }
 
+  private def right[F[_]: Functor, A](a: F[A]): EitherT[F, UploadResult, A] =
+    EitherT.right(a)
+
   def apply[F[_]: Sync](
       store: Store[F],
       queue: JobQueue[F],
@@ -74,10 +96,11 @@ object OUpload {
           notifyJoex: Boolean,
           itemId: Option[Ident]
       ): F[OUpload.UploadResult] =
-        for {
-          files <- data.files.traverse(saveFile).map(_.flatten)
-          pred  <- checkFileList(files)
-          lang  <- store.transact(RCollective.findLanguage(account.collective))
+        (for {
+          _     <- checkExistingItem(itemId, account.collective)
+          files <- right(data.files.traverse(saveFile).map(_.flatten))
+          _     <- checkFileList(files)
+          lang  <- right(store.transact(RCollective.findLanguage(account.collective)))
           meta = ProcessItemArgs.ProcessMeta(
             account.collective,
             itemId,
@@ -89,13 +112,15 @@ object OUpload {
           args =
             if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
             else Vector(ProcessItemArgs(meta, files.toList))
-          job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
-          _   <- logger.fdebug(s"Storing jobs: $job")
-          res <- job.traverse(submitJobs(notifyJoex))
-          _ <- store.transact(
-            RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)
+          jobs <- right(makeJobs(args, account, data.priority, data.tracker))
+          _    <- right(logger.fdebug(s"Storing jobs: $jobs"))
+          res  <- right(submitJobs(notifyJoex)(jobs))
+          _ <- right(
+            store.transact(
+              RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)
+            )
           )
-        } yield res.fold(identity, identity)
+        } yield res).fold(identity, identity)
 
       def submit(
           data: OUpload.UploadData[F],
@@ -111,7 +136,7 @@ object OUpload {
           )
           accId = AccountId(src.cid, src.sid)
           result <- OptionT.liftF(submit(updata, accId, notifyJoex, itemId))
-        } yield result).getOrElse(UploadResult.NoSource)
+        } yield result).getOrElse(UploadResult.noSource)
 
       private def submitJobs(
           notifyJoex: Boolean
@@ -122,6 +147,7 @@ object OUpload {
           _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F]
         } yield UploadResult.Success
 
+      /** Saves the file into the database. */
       private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
         logger.finfo(s"Receiving file $file") *>
           store.bitpeace
@@ -140,10 +166,24 @@ object OUpload {
               )
             )
 
+      private def checkExistingItem(
+          itemId: Option[Ident],
+          coll: Ident
+      ): EitherT[F, UploadResult, Unit] =
+        itemId match {
+          case None =>
+            right(().pure[F])
+          case Some(id) =>
+            OptionT(store.transact(RItem.findByIdAndCollective(id, coll)))
+              .toRight(UploadResult.noItem)
+              .map(_ => ())
+        }
+
       private def checkFileList(
           files: Seq[ProcessItemArgs.File]
-      ): F[Either[UploadResult, Unit]] =
-        Sync[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(()))
+      ): EitherT[F, UploadResult, Unit] =
+        if (files.isEmpty) EitherT.left(UploadResult.noFiles.pure[F])
+        else right(().pure[F])
 
       private def makeJobs(
           args: Vector[ProcessItemArgs],
diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
index 125d6ae0..c27d71d5 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
@@ -470,6 +470,7 @@ trait Conversions {
       case UploadResult.Success  => BasicResult(true, "Files submitted.")
       case UploadResult.NoFiles  => BasicResult(false, "There were no files to submit.")
       case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
+      case UploadResult.NoItem   => BasicResult(false, "The item could not be found.")
     }
 
   def basicResult(cr: PassChangeResult): BasicResult =

From 24caba14578f4efd25397c1070c65e5c0b5684a5 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 11:12:30 +0200
Subject: [PATCH 09/15] Refactor UploadRoutes to remove duplicate code

---
 .../scala/docspell/backend/ops/OUpload.scala  | 13 +++
 .../restserver/routes/UploadRoutes.scala      | 79 ++++++++-----------
 2 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
index 59b73144..0a74c9e5 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -29,6 +29,19 @@ trait OUpload[F[_]] {
       notifyJoex: Boolean,
       itemId: Option[Ident]
   ): F[OUpload.UploadResult]
+
+  final def submitEither(
+      data: OUpload.UploadData[F],
+      accOrSrc: Either[Ident, AccountId],
+      notifyJoex: Boolean,
+      itemId: Option[Ident]
+  ): F[OUpload.UploadResult] =
+    accOrSrc match {
+      case Right(acc) =>
+        submit(data, acc, notifyJoex, itemId)
+      case Left(srcId) =>
+        submit(data, srcId, notifyJoex, itemId)
+    }
 }
 
 object OUpload {
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
index afec48a4..db62a32c 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
@@ -2,13 +2,13 @@ package docspell.restserver.routes
 
 import cats.effect._
 import cats.implicits._
+import docspell.common._
 import docspell.backend.BackendApp
 import docspell.backend.auth.AuthToken
-import docspell.common.{Ident, Priority}
 import docspell.restserver.Config
 import docspell.restserver.conv.Conversions._
 import docspell.restserver.http4s.ResponseGenerator
-import org.http4s.HttpRoutes
+import org.http4s._
 import org.http4s.circe.CirceEntityEncoder._
 import org.http4s.EntityDecoder._
 import org.http4s.dsl.Http4sDsl
@@ -26,32 +26,14 @@ object UploadRoutes {
     val dsl = new Http4sDsl[F] with ResponseGenerator[F] {}
     import dsl._
 
+    val submitting = submitFiles[F](backend, cfg, Right(user.account)) _
+
     HttpRoutes.of {
       case req @ POST -> Root / "item" =>
-        for {
-          multipart <- req.as[Multipart[F]]
-          updata <- readMultipart(
-            multipart,
-            logger,
-            Priority.High,
-            cfg.backend.files.validMimeTypes
-          )
-          result <- backend.upload.submit(updata, user.account, true, None)
-          res    <- Ok(basicResult(result))
-        } yield res
+        submitting(req, None, Priority.High, dsl)
 
       case req @ POST -> Root / "item" / Ident(itemId) =>
-        for {
-          multipart <- req.as[Multipart[F]]
-          updata <- readMultipart(
-            multipart,
-            logger,
-            Priority.High,
-            cfg.backend.files.validMimeTypes
-          )
-          result <- backend.upload.submit(updata, user.account, true, Some(itemId))
-          res    <- Ok(basicResult(result))
-        } yield res
+        submitting(req, Some(itemId), Priority.High, dsl)
     }
   }
 
@@ -61,30 +43,35 @@ object UploadRoutes {
 
     HttpRoutes.of {
       case req @ POST -> Root / "item" / Ident(id) =>
-        for {
-          multipart <- req.as[Multipart[F]]
-          updata <- readMultipart(
-            multipart,
-            logger,
-            Priority.Low,
-            cfg.backend.files.validMimeTypes
-          )
-          result <- backend.upload.submit(updata, id, true, None)
-          res    <- Ok(basicResult(result))
-        } yield res
+        submitFiles(backend, cfg, Left(id))(req, None, Priority.Low, dsl)
 
       case req @ POST -> Root / "item" / Ident(itemId) / Ident(id) =>
-        for {
-          multipart <- req.as[Multipart[F]]
-          updata <- readMultipart(
-            multipart,
-            logger,
-            Priority.Low,
-            cfg.backend.files.validMimeTypes
-          )
-          result <- backend.upload.submit(updata, id, true, Some(itemId))
-          res    <- Ok(basicResult(result))
-        } yield res
+        submitFiles(backend, cfg, Left(id))(req, Some(itemId), Priority.Low, dsl)
     }
   }
+
+  private def submitFiles[F[_]: Effect](
+      backend: BackendApp[F],
+      cfg: Config,
+      accOrSrc: Either[Ident, AccountId]
+  )(
+      req: Request[F],
+      itemId: Option[Ident],
+      prio: Priority,
+      dsl: Http4sDsl[F]
+  ): F[Response[F]] = {
+    import dsl._
+
+    for {
+      multipart <- req.as[Multipart[F]]
+      updata <- readMultipart(
+        multipart,
+        logger,
+        prio,
+        cfg.backend.files.validMimeTypes
+      )
+      result <- backend.upload.submitEither(updata, accOrSrc, true, itemId)
+      res    <- Ok(basicResult(result))
+    } yield res
+  }
 }

From 973847807f2059322e480be9654eab1c97dbba71 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 11:39:20 +0200
Subject: [PATCH 10/15] Update add item view with upload progress info

---
 modules/webapp/src/main/elm/App/Update.elm    |  25 +-
 .../webapp/src/main/elm/Comp/ItemDetail.elm   | 364 ++++++++++--------
 modules/webapp/src/main/elm/Main.elm          |   4 +-
 .../src/main/elm/Page/ItemDetail/Update.elm   |  10 +-
 4 files changed, 226 insertions(+), 177 deletions(-)

diff --git a/modules/webapp/src/main/elm/App/Update.elm b/modules/webapp/src/main/elm/App/Update.elm
index 2abce4df..60a7ffd1 100644
--- a/modules/webapp/src/main/elm/App/Update.elm
+++ b/modules/webapp/src/main/elm/App/Update.elm
@@ -74,7 +74,7 @@ updateWithSub msg model =
             updateNewInvite m model |> noSub
 
         ItemDetailMsg m ->
-            updateItemDetail m model |> noSub
+            updateItemDetail m model
 
         VersionResp (Ok info) ->
             ( { model | version = info }, Cmd.none ) |> noSub
@@ -172,17 +172,20 @@ updateWithSub msg model =
             ( { model | navMenuOpen = not model.navMenuOpen }, Cmd.none, Sub.none )
 
 
-updateItemDetail : Page.ItemDetail.Data.Msg -> Model -> ( Model, Cmd Msg )
+updateItemDetail : Page.ItemDetail.Data.Msg -> Model -> ( Model, Cmd Msg, Sub Msg )
 updateItemDetail lmsg model =
     let
         inav =
             Page.Home.Data.itemNav model.itemDetailModel.detail.item.id model.homeModel
 
-        ( lm, lc ) =
+        ( lm, lc, ls ) =
             Page.ItemDetail.Update.update model.key model.flags inav.next lmsg model.itemDetailModel
     in
-    ( { model | itemDetailModel = lm }
+    ( { model
+        | itemDetailModel = lm
+      }
     , Cmd.map ItemDetailMsg lc
+    , Sub.map ItemDetailMsg ls
     )
 
 
@@ -341,7 +344,19 @@ initPage model page =
             updateQueue Page.Queue.Data.StopRefresh model
 
         ItemDetailPage id ->
-            updateItemDetail (Page.ItemDetail.Data.Init id) model
+            let
+                updateDetail m__ =
+                    let
+                        ( m, c, s ) =
+                            updateItemDetail (Page.ItemDetail.Data.Init id) m__
+                    in
+                    ( { m | subs = Sub.batch [ m.subs, s ] }, c )
+            in
+            Util.Update.andThen1
+                [ updateDetail
+                , updateQueue Page.Queue.Data.StopRefresh
+                ]
+                model
 
 
 noSub : ( Model, Cmd Msg ) -> ( Model, Cmd Msg, Sub Msg )
diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
index 5a883321..b4e32739 100644
--- a/modules/webapp/src/main/elm/Comp/ItemDetail.elm
+++ b/modules/webapp/src/main/elm/Comp/ItemDetail.elm
@@ -393,7 +393,12 @@ isSuccessAll model =
         |> List.all (\id -> Set.member id model.completed)
 
 
-update : Nav.Key -> Flags -> Maybe String -> Msg -> Model -> ( Model, Cmd Msg )
+noSub : ( Model, Cmd Msg ) -> ( Model, Cmd Msg, Sub Msg )
+noSub ( m, c ) =
+    ( m, c, Sub.none )
+
+
+update : Nav.Key -> Flags -> Maybe String -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg )
 update key flags next msg model =
     case msg of
         Init ->
@@ -404,22 +409,23 @@ update key flags next msg model =
                 ( im, ic ) =
                     Comp.ItemMail.init flags
             in
-            ( { model | itemDatePicker = dp, dueDatePicker = dp, itemMail = im, visibleAttach = 0 }
-            , Cmd.batch
-                [ getOptions flags
-                , Cmd.map ItemDatePickerMsg dpc
-                , Cmd.map DueDatePickerMsg dpc
-                , Cmd.map ItemMailMsg ic
-                , Api.getSentMails flags model.item.id SentMailsResp
-                ]
-            )
+            noSub
+                ( { model | itemDatePicker = dp, dueDatePicker = dp, itemMail = im, visibleAttach = 0 }
+                , Cmd.batch
+                    [ getOptions flags
+                    , Cmd.map ItemDatePickerMsg dpc
+                    , Cmd.map DueDatePickerMsg dpc
+                    , Cmd.map ItemMailMsg ic
+                    , Api.getSentMails flags model.item.id SentMailsResp
+                    ]
+                )
 
         SetItem item ->
             let
-                ( m1, c1 ) =
+                ( m1, c1, s1 ) =
                     update key flags next (TagDropdownMsg (Comp.Dropdown.SetSelection item.tags)) model
 
-                ( m2, c2 ) =
+                ( m2, c2, s2 ) =
                     update key
                         flags
                         next
@@ -433,7 +439,7 @@ update key flags next msg model =
                         )
                         m1
 
-                ( m3, c3 ) =
+                ( m3, c3, s3 ) =
                     update key
                         flags
                         next
@@ -447,7 +453,7 @@ update key flags next msg model =
                         )
                         m2
 
-                ( m4, c4 ) =
+                ( m4, c4, s4 ) =
                     update key
                         flags
                         next
@@ -461,7 +467,7 @@ update key flags next msg model =
                         )
                         m3
 
-                ( m5, c5 ) =
+                ( m5, c5, s5 ) =
                     update key
                         flags
                         next
@@ -475,7 +481,7 @@ update key flags next msg model =
                         )
                         m4
 
-                ( m6, c6 ) =
+                ( m6, c6, s6 ) =
                     update key
                         flags
                         next
@@ -489,7 +495,7 @@ update key flags next msg model =
                         )
                         m5
 
-                ( m7, c7 ) =
+                ( m7, c7, s7 ) =
                     update key flags next AddFilesReset m6
 
                 proposalCmd =
@@ -520,20 +526,21 @@ update key flags next msg model =
                 , proposalCmd
                 , Api.getSentMails flags item.id SentMailsResp
                 ]
+            , Sub.batch [ s1, s2, s3, s4, s5, s6, s7 ]
             )
 
         SetActiveAttachment pos ->
-            ( { model | visibleAttach = pos, sentMailsOpen = False }, Cmd.none )
+            noSub ( { model | visibleAttach = pos, sentMailsOpen = False }, Cmd.none )
 
         ToggleMenu ->
-            ( { model | menuOpen = not model.menuOpen }, Cmd.none )
+            noSub ( { model | menuOpen = not model.menuOpen }, Cmd.none )
 
         ReloadItem ->
             if model.item.id == "" then
-                ( model, Cmd.none )
+                noSub ( model, Cmd.none )
 
             else
-                ( model, Api.itemDetail flags model.item.id GetItemResp )
+                noSub ( model, Api.itemDetail flags model.item.id GetItemResp )
 
         TagDropdownMsg m ->
             let
@@ -550,7 +557,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map TagDropdownMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map TagDropdownMsg c2 ] )
 
         DirDropdownMsg m ->
             let
@@ -567,7 +574,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map DirDropdownMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map DirDropdownMsg c2 ] )
 
         OrgDropdownMsg m ->
             let
@@ -587,7 +594,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map OrgDropdownMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map OrgDropdownMsg c2 ] )
 
         CorrPersonMsg m ->
             let
@@ -607,7 +614,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map CorrPersonMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map CorrPersonMsg c2 ] )
 
         ConcPersonMsg m ->
             let
@@ -627,7 +634,7 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map ConcPersonMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map ConcPersonMsg c2 ] )
 
         ConcEquipMsg m ->
             let
@@ -647,42 +654,45 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( newModel, Cmd.batch [ save, Cmd.map ConcEquipMsg c2 ] )
+            noSub ( newModel, Cmd.batch [ save, Cmd.map ConcEquipMsg c2 ] )
 
         SetName str ->
-            ( { model | nameModel = str }, Cmd.none )
+            noSub ( { model | nameModel = str }, Cmd.none )
 
         SaveName ->
-            ( model, setName flags model )
+            noSub ( model, setName flags model )
 
         SetNotes str ->
-            ( { model | notesModel = Util.Maybe.fromString str }
-            , Cmd.none
-            )
+            noSub
+                ( { model | notesModel = Util.Maybe.fromString str }
+                , Cmd.none
+                )
 
         ToggleNotes ->
-            ( { model
-                | notesField =
-                    if model.notesField == ViewNotes then
-                        HideNotes
+            noSub
+                ( { model
+                    | notesField =
+                        if model.notesField == ViewNotes then
+                            HideNotes
 
-                    else
-                        ViewNotes
-              }
-            , Cmd.none
-            )
+                        else
+                            ViewNotes
+                  }
+                , Cmd.none
+                )
 
         ToggleEditNotes ->
-            ( { model
-                | notesField =
-                    if isEditNotes model.notesField then
-                        ViewNotes
+            noSub
+                ( { model
+                    | notesField =
+                        if isEditNotes model.notesField then
+                            ViewNotes
 
-                    else
-                        EditNotes Comp.MarkdownInput.init
-              }
-            , Cmd.none
-            )
+                        else
+                            EditNotes Comp.MarkdownInput.init
+                  }
+                , Cmd.none
+                )
 
         NotesEditMsg lm ->
             case model.notesField of
@@ -691,21 +701,25 @@ update key flags next msg model =
                         ( lm2, str ) =
                             Comp.MarkdownInput.update (Maybe.withDefault "" model.notesModel) lm em
                     in
-                    ( { model | notesField = EditNotes lm2, notesModel = Util.Maybe.fromString str }
-                    , Cmd.none
-                    )
+                    noSub
+                        ( { model | notesField = EditNotes lm2, notesModel = Util.Maybe.fromString str }
+                        , Cmd.none
+                        )
 
-                _ ->
-                    ( model, Cmd.none )
+                HideNotes ->
+                    noSub ( model, Cmd.none )
+
+                ViewNotes ->
+                    noSub ( model, Cmd.none )
 
         SaveNotes ->
-            ( model, setNotes flags model )
+            noSub ( model, setNotes flags model )
 
         ConfirmItem ->
-            ( model, Api.setConfirmed flags model.item.id SaveResp )
+            noSub ( model, Api.setConfirmed flags model.item.id SaveResp )
 
         UnconfirmItem ->
-            ( model, Api.setUnconfirmed flags model.item.id SaveResp )
+            noSub ( model, Api.setUnconfirmed flags model.item.id SaveResp )
 
         ItemDatePickerMsg m ->
             let
@@ -718,13 +732,13 @@ update key flags next msg model =
                         newModel =
                             { model | itemDatePicker = dp, itemDate = Just (Comp.DatePicker.midOfDay date) }
                     in
-                    ( newModel, setDate flags newModel newModel.itemDate )
+                    noSub ( newModel, setDate flags newModel newModel.itemDate )
 
                 _ ->
-                    ( { model | itemDatePicker = dp }, Cmd.none )
+                    noSub ( { model | itemDatePicker = dp }, Cmd.none )
 
         RemoveDate ->
-            ( { model | itemDate = Nothing }, setDate flags model Nothing )
+            noSub ( { model | itemDate = Nothing }, setDate flags model Nothing )
 
         DueDatePickerMsg m ->
             let
@@ -737,13 +751,13 @@ update key flags next msg model =
                         newModel =
                             { model | dueDatePicker = dp, dueDate = Just (Comp.DatePicker.midOfDay date) }
                     in
-                    ( newModel, setDueDate flags newModel newModel.dueDate )
+                    noSub ( newModel, setDueDate flags newModel newModel.dueDate )
 
                 _ ->
-                    ( { model | dueDatePicker = dp }, Cmd.none )
+                    noSub ( { model | dueDatePicker = dp }, Cmd.none )
 
         RemoveDueDate ->
-            ( { model | dueDate = Nothing }, setDueDate flags model Nothing )
+            noSub ( { model | dueDate = Nothing }, setDueDate flags model Nothing )
 
         DeleteItemConfirm m ->
             let
@@ -757,41 +771,41 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( { model | deleteItemConfirm = cm }, cmd )
+            noSub ( { model | deleteItemConfirm = cm }, cmd )
 
         RequestDelete ->
             update key flags next (DeleteItemConfirm Comp.YesNoDimmer.activate) model
 
         SetCorrOrgSuggestion idname ->
-            ( model, setCorrOrg flags model (Just idname) )
+            noSub ( model, setCorrOrg flags model (Just idname) )
 
         SetCorrPersonSuggestion idname ->
-            ( model, setCorrPerson flags model (Just idname) )
+            noSub ( model, setCorrPerson flags model (Just idname) )
 
         SetConcPersonSuggestion idname ->
-            ( model, setConcPerson flags model (Just idname) )
+            noSub ( model, setConcPerson flags model (Just idname) )
 
         SetConcEquipSuggestion idname ->
-            ( model, setConcEquip flags model (Just idname) )
+            noSub ( model, setConcEquip flags model (Just idname) )
 
         SetItemDateSuggestion date ->
-            ( model, setDate flags model (Just date) )
+            noSub ( model, setDate flags model (Just date) )
 
         SetDueDateSuggestion date ->
-            ( model, setDueDate flags model (Just date) )
+            noSub ( model, setDueDate flags model (Just date) )
 
         GetTagsResp (Ok tags) ->
             let
                 tagList =
                     Comp.Dropdown.SetOptions tags.items
 
-                ( m1, c1 ) =
+                ( m1, c1, s1 ) =
                     update key flags next (TagDropdownMsg tagList) model
             in
-            ( m1, c1 )
+            ( m1, c1, s1 )
 
         GetTagsResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         GetOrgResp (Ok orgs) ->
             let
@@ -801,23 +815,23 @@ update key flags next msg model =
             update key flags next (OrgDropdownMsg opts) model
 
         GetOrgResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         GetPersonResp (Ok ps) ->
             let
                 opts =
                     Comp.Dropdown.SetOptions ps.items
 
-                ( m1, c1 ) =
+                ( m1, c1, s1 ) =
                     update key flags next (CorrPersonMsg opts) model
 
-                ( m2, c2 ) =
+                ( m2, c2, s2 ) =
                     update key flags next (ConcPersonMsg opts) m1
             in
-            ( m2, Cmd.batch [ c1, c2 ] )
+            ( m2, Cmd.batch [ c1, c2 ], Sub.batch [ s1, s2 ] )
 
         GetPersonResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         GetEquipResp (Ok equips) ->
             let
@@ -830,44 +844,44 @@ update key flags next msg model =
             update key flags next (ConcEquipMsg opts) model
 
         GetEquipResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         SaveResp (Ok res) ->
             if res.success then
-                ( model, Api.itemDetail flags model.item.id GetItemResp )
+                noSub ( model, Api.itemDetail flags model.item.id GetItemResp )
 
             else
-                ( model, Cmd.none )
+                noSub ( model, Cmd.none )
 
         SaveResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         DeleteResp (Ok res) ->
             if res.success then
                 case next of
                     Just id ->
-                        ( model, Page.set key (ItemDetailPage id) )
+                        noSub ( model, Page.set key (ItemDetailPage id) )
 
                     Nothing ->
-                        ( model, Page.set key HomePage )
+                        noSub ( model, Page.set key HomePage )
 
             else
-                ( model, Cmd.none )
+                noSub ( model, Cmd.none )
 
         DeleteResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         GetItemResp (Ok item) ->
             update key flags next (SetItem item) model
 
         GetItemResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         GetProposalResp (Ok ip) ->
-            ( { model | itemProposals = ip }, Cmd.none )
+            noSub ( { model | itemProposals = ip }, Cmd.none )
 
         GetProposalResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         ItemMailMsg m ->
             let
@@ -876,16 +890,17 @@ update key flags next msg model =
             in
             case fa of
                 Comp.ItemMail.FormNone ->
-                    ( { model | itemMail = im }, Cmd.map ItemMailMsg ic )
+                    noSub ( { model | itemMail = im }, Cmd.map ItemMailMsg ic )
 
                 Comp.ItemMail.FormCancel ->
-                    ( { model
-                        | itemMail = Comp.ItemMail.clear im
-                        , mailOpen = False
-                        , mailSendResult = Nothing
-                      }
-                    , Cmd.map ItemMailMsg ic
-                    )
+                    noSub
+                        ( { model
+                            | itemMail = Comp.ItemMail.clear im
+                            , mailOpen = False
+                            , mailSendResult = Nothing
+                          }
+                        , Cmd.map ItemMailMsg ic
+                        )
 
                 Comp.ItemMail.FormSend sm ->
                     let
@@ -895,12 +910,13 @@ update key flags next msg model =
                             , conn = sm.conn
                             }
                     in
-                    ( { model | mailSending = True }
-                    , Cmd.batch
-                        [ Cmd.map ItemMailMsg ic
-                        , Api.sendMail flags mail SendMailResp
-                        ]
-                    )
+                    noSub
+                        ( { model | mailSending = True }
+                        , Cmd.batch
+                            [ Cmd.map ItemMailMsg ic
+                            , Api.sendMail flags mail SendMailResp
+                            ]
+                        )
 
         ToggleMail ->
             let
@@ -914,12 +930,13 @@ update key flags next msg model =
                     else
                         Nothing
             in
-            ( { model
-                | mailOpen = newOpen
-                , mailSendResult = sendResult
-              }
-            , Cmd.none
-            )
+            noSub
+                ( { model
+                    | mailOpen = newOpen
+                    , mailSendResult = sendResult
+                  }
+                , Cmd.none
+                )
 
         SendMailResp (Ok br) ->
             let
@@ -930,56 +947,59 @@ update key flags next msg model =
                     else
                         model.itemMail
             in
-            ( { model
-                | itemMail = mm
-                , mailSending = False
-                , mailSendResult = Just br
-              }
-            , if br.success then
-                Api.itemDetail flags model.item.id GetItemResp
+            noSub
+                ( { model
+                    | itemMail = mm
+                    , mailSending = False
+                    , mailSendResult = Just br
+                  }
+                , if br.success then
+                    Api.itemDetail flags model.item.id GetItemResp
 
-              else
-                Cmd.none
-            )
+                  else
+                    Cmd.none
+                )
 
         SendMailResp (Err err) ->
             let
                 errmsg =
                     Util.Http.errorToString err
             in
-            ( { model
-                | mailSendResult = Just (BasicResult False errmsg)
-                , mailSending = False
-              }
-            , Cmd.none
-            )
+            noSub
+                ( { model
+                    | mailSendResult = Just (BasicResult False errmsg)
+                    , mailSending = False
+                  }
+                , Cmd.none
+                )
 
         SentMailsMsg m ->
             let
                 sm =
                     Comp.SentMails.update m model.sentMails
             in
-            ( { model | sentMails = sm }, Cmd.none )
+            noSub ( { model | sentMails = sm }, Cmd.none )
 
         ToggleSentMails ->
-            ( { model | sentMailsOpen = not model.sentMailsOpen, visibleAttach = -1 }, Cmd.none )
+            noSub ( { model | sentMailsOpen = not model.sentMailsOpen, visibleAttach = -1 }, Cmd.none )
 
         SentMailsResp (Ok list) ->
             let
                 sm =
                     Comp.SentMails.initMails list.items
             in
-            ( { model | sentMails = sm }, Cmd.none )
+            noSub ( { model | sentMails = sm }, Cmd.none )
 
         SentMailsResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         AttachMetaClick id ->
             case Dict.get id model.attachMeta of
                 Just _ ->
-                    ( { model | attachMetaOpen = not model.attachMetaOpen }
-                    , Cmd.none
-                    )
+                    noSub
+                        ( { model | attachMetaOpen = not model.attachMetaOpen }
+                        , Cmd.none
+                        )
 
                 Nothing ->
                     let
@@ -989,9 +1009,10 @@ update key flags next msg model =
                         nextMeta =
                             Dict.insert id am model.attachMeta
                     in
-                    ( { model | attachMeta = nextMeta, attachMetaOpen = True }
-                    , Cmd.map (AttachMetaMsg id) ac
-                    )
+                    noSub
+                        ( { model | attachMeta = nextMeta, attachMetaOpen = True }
+                        , Cmd.map (AttachMetaMsg id) ac
+                        )
 
         AttachMetaMsg id lmsg ->
             case Dict.get id model.attachMeta of
@@ -1000,17 +1021,19 @@ update key flags next msg model =
                         am =
                             Comp.AttachmentMeta.update lmsg cm
                     in
-                    ( { model | attachMeta = Dict.insert id am model.attachMeta }
-                    , Cmd.none
-                    )
+                    noSub
+                        ( { model | attachMeta = Dict.insert id am model.attachMeta }
+                        , Cmd.none
+                        )
 
                 Nothing ->
-                    ( model, Cmd.none )
+                    noSub ( model, Cmd.none )
 
         TogglePdfNativeView ->
-            ( { model | pdfNativeView = not model.pdfNativeView }
-            , Cmd.none
-            )
+            noSub
+                ( { model | pdfNativeView = not model.pdfNativeView }
+                , Cmd.none
+                )
 
         DeleteAttachConfirm attachId lmsg ->
             let
@@ -1024,17 +1047,17 @@ update key flags next msg model =
                     else
                         Cmd.none
             in
-            ( { model | deleteAttachConfirm = cm }, cmd )
+            noSub ( { model | deleteAttachConfirm = cm }, cmd )
 
         DeleteAttachResp (Ok res) ->
             if res.success then
                 update key flags next ReloadItem model
 
             else
-                ( model, Cmd.none )
+                noSub ( model, Cmd.none )
 
         DeleteAttachResp (Err _) ->
-            ( model, Cmd.none )
+            noSub ( model, Cmd.none )
 
         RequestDeleteAttachment id ->
             update key
@@ -1044,9 +1067,10 @@ update key flags next msg model =
                 model
 
         AddFilesToggle ->
-            ( { model | addFilesOpen = not model.addFilesOpen }
-            , Cmd.none
-            )
+            noSub
+                ( { model | addFilesOpen = not model.addFilesOpen }
+                , Cmd.none
+                )
 
         AddFilesMsg lm ->
             let
@@ -1056,20 +1080,22 @@ update key flags next msg model =
                 nextFiles =
                     model.selectedFiles ++ df
             in
-            ( { model | addFilesModel = dm, selectedFiles = nextFiles }
-            , Cmd.map AddFilesMsg dc
-            )
+            noSub
+                ( { model | addFilesModel = dm, selectedFiles = nextFiles }
+                , Cmd.map AddFilesMsg dc
+                )
 
         AddFilesReset ->
-            ( { model
-                | selectedFiles = []
-                , addFilesModel = Comp.Dropzone.init Comp.Dropzone.defaultSettings
-                , completed = Set.empty
-                , errored = Set.empty
-                , loading = Set.empty
-              }
-            , Cmd.none
-            )
+            noSub
+                ( { model
+                    | selectedFiles = []
+                    , addFilesModel = Comp.Dropzone.init Comp.Dropzone.defaultSettings
+                    , completed = Set.empty
+                    , errored = Set.empty
+                    , loading = Set.empty
+                  }
+                , Cmd.none
+                )
 
         AddFilesSubmitUpload ->
             let
@@ -1085,7 +1111,10 @@ update key flags next msg model =
                 ( cm2, _, _ ) =
                     Comp.Dropzone.update (Comp.Dropzone.setActive False) model.addFilesModel
             in
-            ( { model | loading = Set.fromList fileids, addFilesModel = cm2 }, uploads )
+            ( { model | loading = Set.fromList fileids, addFilesModel = cm2 }
+            , uploads
+            , tracker
+            )
 
         AddFilesUploadResp fileid (Ok res) ->
             let
@@ -1109,9 +1138,10 @@ update key flags next msg model =
                 newModel =
                     { model | completed = compl, errored = errs, loading = load }
             in
-            ( newModel
-            , Ports.setProgress ( fileid, 100 )
-            )
+            noSub
+                ( newModel
+                , Ports.setProgress ( fileid, 100 )
+                )
 
         AddFilesUploadResp fileid (Err _) ->
             let
@@ -1121,7 +1151,7 @@ update key flags next msg model =
                 load =
                     Set.remove fileid model.loading
             in
-            ( { model | errored = errs, loading = load }, Cmd.none )
+            noSub ( { model | errored = errs, loading = load }, Cmd.none )
 
         AddFilesProgress fileid progress ->
             let
@@ -1142,7 +1172,7 @@ update key flags next msg model =
                     else
                         Ports.setProgress ( fileid, percent )
             in
-            ( model, updateBars )
+            noSub ( model, updateBars )
 
 
 
diff --git a/modules/webapp/src/main/elm/Main.elm b/modules/webapp/src/main/elm/Main.elm
index 1d2f45d2..0206cee9 100644
--- a/modules/webapp/src/main/elm/Main.elm
+++ b/modules/webapp/src/main/elm/Main.elm
@@ -58,7 +58,9 @@ init flags url key =
                 Nothing ->
                     Cmd.none
     in
-    ( m, Cmd.batch [ cmd, Api.versionInfo flags VersionResp, sessionCheck ] )
+    ( m
+    , Cmd.batch [ cmd, Api.versionInfo flags VersionResp, sessionCheck ]
+    )
 
 
 viewDoc : Model -> Document Msg
diff --git a/modules/webapp/src/main/elm/Page/ItemDetail/Update.elm b/modules/webapp/src/main/elm/Page/ItemDetail/Update.elm
index 6c916d67..60ca8453 100644
--- a/modules/webapp/src/main/elm/Page/ItemDetail/Update.elm
+++ b/modules/webapp/src/main/elm/Page/ItemDetail/Update.elm
@@ -7,25 +7,27 @@ import Data.Flags exposing (Flags)
 import Page.ItemDetail.Data exposing (Model, Msg(..))
 
 
-update : Nav.Key -> Flags -> Maybe String -> Msg -> Model -> ( Model, Cmd Msg )
+update : Nav.Key -> Flags -> Maybe String -> Msg -> Model -> ( Model, Cmd Msg, Sub Msg )
 update key flags next msg model =
     case msg of
         Init id ->
             let
-                ( lm, lc ) =
+                ( lm, lc, ls ) =
                     Comp.ItemDetail.update key flags next Comp.ItemDetail.Init model.detail
             in
             ( { model | detail = lm }
             , Cmd.batch [ Api.itemDetail flags id ItemResp, Cmd.map ItemDetailMsg lc ]
+            , Sub.map ItemDetailMsg ls
             )
 
         ItemDetailMsg lmsg ->
             let
-                ( lm, lc ) =
+                ( lm, lc, ls ) =
                     Comp.ItemDetail.update key flags next lmsg model.detail
             in
             ( { model | detail = lm }
             , Cmd.map ItemDetailMsg lc
+            , Sub.map ItemDetailMsg ls
             )
 
         ItemResp (Ok item) ->
@@ -36,4 +38,4 @@ update key flags next msg model =
             update key flags next (ItemDetailMsg lmsg) model
 
         ItemResp (Err _) ->
-            ( model, Cmd.none )
+            ( model, Cmd.none, Sub.none )

From 938866fb845274b07f03b1f841bb87fb60404198 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 12:04:39 +0200
Subject: [PATCH 11/15] Update changelog

---
 Changelog.md | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index 18878afa..6e7165a2 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -8,18 +8,21 @@
   periodically to import your mails.
 - New feature "Integration Endpoint". Allows an admin to upload files
   to any collective using a separate endpoint.
+- New feature: add files to existing items.
+- The document list on the front-page has been rewritten. The table is
+  removed and documents are now presented in a “card view”.
+- Amend the mail-to-pdf conversion to include the e-mail date.
+- When processing e-mails, set the item date automatically from the
+  received-date in the mail.
+- Fixes regarding character encodings when reading e-mails.
 - Fix the `find-by-checksum` route that, given a sha256 checksum,
   returns whether there is such a file in docspell. It falsely
   returned `false` although documents existed.
-- Amend the mail-to-pdf conversion to include the e-mail date.
 - Fix webapp for mobile devices.
-- The document list on the front-page has been rewritten. The table is
-  removed and documents are now presented in a “card view”.
 - Fix the search menu to remember dates in fields. When going back
   from an item detail to the front-page, the search menu remembers the
   last state, but dates were cleared.
-- More fixes regarding character encodings when reading e-mails.
-- Fix redirecting `/` to `/app`.
+- Fix redirecting `/` only to `/app`.
 
 ### Configuration Changes
 

From 4e49c78e724190830d28b857b01e036b2b63f0b7 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 12:54:35 +0200
Subject: [PATCH 12/15] Change some log levels of item processing task

---
 .../joex/src/main/scala/docspell/joex/process/ConvertPdf.scala  | 2 +-
 .../src/main/scala/docspell/joex/process/TextExtraction.scala   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala
index 5435ee69..3aba96dd 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala
@@ -62,7 +62,7 @@ object ConvertPdf {
     Conversion.create[F](cfg, sanitizeHtml, ctx.blocker, ctx.logger).use { conv =>
       mime match {
         case mt if mt.baseEqual(Mimetype.`application/pdf`) =>
-          ctx.logger.info("Not going to convert a PDF file into a PDF.") *>
+          ctx.logger.debug(s"Not going to convert a PDF file ${ra.name} into a PDF.") *>
             (ra, None: Option[RAttachmentMeta]).pure[F]
 
         case _ =>
diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
index 966a0986..0df63258 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
@@ -60,7 +60,7 @@ object TextExtraction {
         rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty))
       )
       est <- dst
-      _ <- ctx.logger.debug(
+      _ <- ctx.logger.info(
         s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
       )
     } yield meta

From 1dde43e09275774717d4ed52bec4718da5c6c97e Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 13:28:21 +0200
Subject: [PATCH 13/15] Only process attachments in task arguments

When files are added to an item, the attachments already present must
not be "re-processed".
---
 .../src/main/scala/docspell/joex/process/CreateItem.scala | 8 ++++++--
 .../src/main/scala/docspell/store/queries/QItem.scala     | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
index 30737bba..8ef30826 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
@@ -119,8 +119,9 @@ object CreateItem {
 
   private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] =
     Task { ctx =>
+      val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet
       for {
-        cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
+        cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq))
         _ <-
           if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
           else ().pure[F]
@@ -130,8 +131,11 @@ object CreateItem {
             ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
           else ().pure[F]
         rms <- OptionT(
+          //load attachments but only those mentioned in the task's arguments
           cand.headOption.traverse(ri =>
-            ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
+            ctx.store
+              .transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
+              .map(_.filter(r => fileMetaIds.contains(r.fileId)))
           )
         ).getOrElse(Vector.empty)
         orig <- rms.traverse(a =>
diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala
index 6230a3bb..619c0dbf 100644
--- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala
@@ -287,7 +287,7 @@ object QItem {
       n  <- store.transact(RItem.deleteByIdAndCollective(itemId, collective))
     } yield tn + rn + n
 
-  def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = {
+  def findByFileIds(fileMetaIds: Seq[Ident]): ConnectionIO[Vector[RItem]] = {
     val IC = RItem.Columns
     val AC = RAttachment.Columns
     val q =

From 4694433e38b656c383ba658066750c8ba3bb9904 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 14:58:18 +0200
Subject: [PATCH 14/15] Fix attachment positions

It worked for new items, because the implicit offset was 0. when
adding archives to existing items, there are already attachments and
the new attachments are added to the end. This won't work if files are
added concurrently, because there is no quick and reliable way to
determine the offset then.
---
 .../docspell/joex/process/CreateItem.scala    |   2 +-
 .../joex/process/ExtractArchive.scala         | 118 +++++++++++++-----
 .../scala/docspell/store/impl/Column.scala    |   2 +
 .../docspell/store/records/RAttachment.scala  |   6 +-
 4 files changed, 95 insertions(+), 33 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
index 8ef30826..0a06b421 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
@@ -32,7 +32,7 @@ object CreateItem {
 
       def fileMetas(itemId: Ident, now: Timestamp) =
         Stream
-          .eval(ctx.store.transact(RAttachment.countOnItem(itemId)))
+          .eval(ctx.store.transact(RAttachment.nextPosition(itemId)))
           .flatMap { offset =>
             Stream
               .emits(ctx.args.files)
diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
index d3a156ff..06cbba72 100644
--- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
+++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala
@@ -13,6 +13,8 @@ import docspell.store.records._
 import docspell.files.Zip
 import cats.kernel.Monoid
 import emil.Mail
+import cats.kernel.Order
+import cats.data.NonEmptyList
 
 /** Goes through all attachments and extracts archive files, like zip
   * files. The process is recursive, until all archives have been
@@ -46,24 +48,37 @@ object ExtractArchive {
       archive: Option[RAttachmentArchive]
   ): Task[F, ProcessItemArgs, (Option[RAttachmentArchive], ItemData)] =
     Task { ctx =>
-      def extract(ra: RAttachment) =
-        findMime(ctx)(ra).flatMap(m => extractSafe(ctx, archive)(ra, m))
+      def extract(ra: RAttachment, pos: Int): F[Extracted] =
+        findMime(ctx)(ra).flatMap(m => extractSafe(ctx, archive)(ra, pos, m))
 
       for {
-        ras     <- item.attachments.traverse(extract)
-        lastPos <- ctx.store.transact(RAttachment.countOnItem(item.item.id))
-        nra =
-          ras.flatMap(_.files).zipWithIndex.map(t => t._1.copy(position = lastPos + t._2))
-        _ <- nra.traverse(storeAttachment(ctx))
-        naa = ras.flatMap(_.archives)
+        lastPos <- ctx.store.transact(RAttachment.nextPosition(item.item.id))
+        extracts <-
+          item.attachments.zipWithIndex
+            .traverse(t => extract(t._1, lastPos + t._2))
+            .map(Monoid[Extracted].combineAll)
+            .map(fixPositions)
+        nra = extracts.files
+        _ <- extracts.files.traverse(storeAttachment(ctx))
+        naa = extracts.archives
         _ <- naa.traverse(storeArchive(ctx))
       } yield naa.headOption -> item.copy(
         attachments = nra,
         originFile = item.originFile ++ nra.map(a => a.id -> a.fileId).toMap,
-        givenMeta = item.givenMeta.fillEmptyFrom(Monoid[Extracted].combineAll(ras).meta)
+        givenMeta = item.givenMeta.fillEmptyFrom(extracts.meta)
       )
     }
 
+  /** After all files have been extracted, the `extract' contains the
+    * whole (combined) result. This fixes positions of the attachments
+    * such that the elements of an archive are "spliced" into the
+    * attachment list at the position of the archive. If there is no
+    * archive, positions don't need to be fixed.
+    */
+  private def fixPositions(extract: Extracted): Extracted =
+    if (extract.archives.isEmpty) extract
+    else extract.updatePositions
+
   def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[Mimetype] =
     OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId)))
       .map(_.mimetype)
@@ -72,21 +87,21 @@ object ExtractArchive {
   def extractSafe[F[_]: ConcurrentEffect: ContextShift](
       ctx: Context[F, ProcessItemArgs],
       archive: Option[RAttachmentArchive]
-  )(ra: RAttachment, mime: Mimetype): F[Extracted] =
+  )(ra: RAttachment, pos: Int, mime: Mimetype): F[Extracted] =
     mime match {
       case Mimetype("application", "zip", _) if ra.name.exists(_.endsWith(".zip")) =>
         ctx.logger.info(s"Extracting zip archive ${ra.name.getOrElse("<noname>")}.") *>
-          extractZip(ctx, archive)(ra)
+          extractZip(ctx, archive)(ra, pos)
             .flatTap(_ => cleanupParents(ctx, ra, archive))
 
       case Mimetype("message", "rfc822", _) =>
         ctx.logger.info(s"Reading e-mail ${ra.name.getOrElse("<noname>")}") *>
-          extractMail(ctx, archive)(ra)
+          extractMail(ctx, archive)(ra, pos)
             .flatTap(_ => cleanupParents(ctx, ra, archive))
 
       case _ =>
         ctx.logger.debug(s"Not an archive: ${mime.asString}") *>
-          Extracted.noArchive(ra).pure[F]
+          Extracted.noArchive(ra, pos, 0).pure[F]
     }
 
   def cleanupParents[F[_]: Sync](
@@ -116,7 +131,7 @@ object ExtractArchive {
   def extractZip[F[_]: ConcurrentEffect: ContextShift](
       ctx: Context[F, _],
       archive: Option[RAttachmentArchive]
-  )(ra: RAttachment): F[Extracted] = {
+  )(ra: RAttachment, pos: Int): F[Extracted] = {
     val zipData = ctx.store.bitpeace
       .get(ra.fileId.id)
       .unNoneTerminate
@@ -124,7 +139,8 @@ object ExtractArchive {
 
     zipData
       .through(Zip.unzipP[F](8192, ctx.blocker))
-      .flatMap(handleEntry(ctx, ra, archive, None))
+      .zipWithIndex
+      .flatMap(handleEntry(ctx, ra, pos, archive, None))
       .foldMonoid
       .compile
       .lastOrError
@@ -133,7 +149,7 @@ object ExtractArchive {
   def extractMail[F[_]: ConcurrentEffect: ContextShift](
       ctx: Context[F, _],
       archive: Option[RAttachmentArchive]
-  )(ra: RAttachment): F[Extracted] = {
+  )(ra: RAttachment, pos: Int): F[Extracted] = {
     val email: Stream[F, Byte] = ctx.store.bitpeace
       .get(ra.fileId.id)
       .unNoneTerminate
@@ -151,7 +167,8 @@ object ExtractArchive {
 
         ReadMail
           .mailToEntries(ctx.logger)(mail)
-          .flatMap(handleEntry(ctx, ra, archive, mId)) ++ Stream.eval(givenMeta)
+          .zipWithIndex
+          .flatMap(handleEntry(ctx, ra, pos, archive, mId)) ++ Stream.eval(givenMeta)
       }
       .foldMonoid
       .compile
@@ -167,13 +184,15 @@ object ExtractArchive {
   def handleEntry[F[_]: Sync](
       ctx: Context[F, _],
       ra: RAttachment,
+      pos: Int,
       archive: Option[RAttachmentArchive],
       messageId: Option[String]
   )(
-      entry: Binary[F]
+      tentry: (Binary[F], Long)
   ): Stream[F, Extracted] = {
-    val mimeHint = MimetypeHint.filename(entry.name).withAdvertised(entry.mime.asString)
-    val fileMeta = ctx.store.bitpeace.saveNew(entry.data, 8192, mimeHint)
+    val (entry, subPos) = tentry
+    val mimeHint        = MimetypeHint.filename(entry.name).withAdvertised(entry.mime.asString)
+    val fileMeta        = ctx.store.bitpeace.saveNew(entry.data, 8192, mimeHint)
     Stream.eval(ctx.logger.debug(s"Extracted ${entry.name}. Storing as attachment.")) >>
       fileMeta.evalMap { fm =>
         Ident.randomId.map { id =>
@@ -181,12 +200,12 @@ object ExtractArchive {
             id,
             ra.itemId,
             Ident.unsafe(fm.id),
-            0, //position is updated afterwards
+            pos,
             ra.created,
             Option(entry.name).map(_.trim).filter(_.nonEmpty)
           )
           val aa = archive.getOrElse(RAttachmentArchive.of(ra, messageId)).copy(id = id)
-          Extracted.of(nra, aa)
+          Extracted.of(nra, aa, pos, subPos.toInt)
         }
       }
 
@@ -206,28 +225,67 @@ object ExtractArchive {
   case class Extracted(
       files: Vector[RAttachment],
       archives: Vector[RAttachmentArchive],
-      meta: MetaProposalList
+      meta: MetaProposalList,
+      positions: List[Extracted.Pos]
   ) {
     def ++(e: Extracted) =
-      Extracted(files ++ e.files, archives ++ e.archives, meta.fillEmptyFrom(e.meta))
+      Extracted(
+        files ++ e.files,
+        archives ++ e.archives,
+        meta.fillEmptyFrom(e.meta),
+        positions ++ e.positions
+      )
 
     def setMeta(m: MetaProposal): Extracted =
       setMeta(MetaProposalList.of(m))
 
     def setMeta(ml: MetaProposalList): Extracted =
-      Extracted(files, archives, meta.fillEmptyFrom(ml))
+      Extracted(files, archives, meta.fillEmptyFrom(ml), positions)
+
+    def updatePositions: Extracted =
+      NonEmptyList.fromList(positions) match {
+        case None =>
+          this
+        case Some(nel) =>
+          val sorted = nel.sorted
+          println(s"---------------------------- $sorted ")
+          val offset = sorted.head.first
+          val pos =
+            sorted.zipWithIndex.map({ case (p, i) => p.id -> (i + offset) }).toList.toMap
+          val nf =
+            files.map(f => pos.get(f.id).map(n => f.copy(position = n)).getOrElse(f))
+          copy(files = nf)
+      }
   }
   object Extracted {
-    val empty = Extracted(Vector.empty, Vector.empty, MetaProposalList.empty)
+    val empty =
+      Extracted(Vector.empty, Vector.empty, MetaProposalList.empty, Nil)
 
-    def noArchive(ra: RAttachment): Extracted =
-      Extracted(Vector(ra), Vector.empty, MetaProposalList.empty)
+    def noArchive(ra: RAttachment, pos: Int, subPos: Int): Extracted =
+      Extracted(
+        Vector(ra),
+        Vector.empty,
+        MetaProposalList.empty,
+        List(Pos(ra.id, pos, subPos))
+      )
 
-    def of(ra: RAttachment, aa: RAttachmentArchive): Extracted =
-      Extracted(Vector(ra), Vector(aa), MetaProposalList.empty)
+    def of(ra: RAttachment, aa: RAttachmentArchive, pos: Int, subPos: Int): Extracted =
+      Extracted(
+        Vector(ra),
+        Vector(aa),
+        MetaProposalList.empty,
+        List(Pos(ra.id, pos, subPos))
+      )
 
     implicit val extractedMonoid: Monoid[Extracted] =
       Monoid.instance(empty, _ ++ _)
+
+    case class Pos(id: Ident, first: Int, second: Int)
+
+    object Pos {
+      implicit val ordering: Order[Pos] =
+        Order.whenEqual(Order.by(_.first), Order.by(_.second))
+    }
   }
 
 }
diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala
index d84ed3cf..0cdc0be3 100644
--- a/modules/store/src/main/scala/docspell/store/impl/Column.scala
+++ b/modules/store/src/main/scala/docspell/store/impl/Column.scala
@@ -101,4 +101,6 @@ case class Column(name: String, ns: String = "", alias: String = "") {
   def asc: Fragment =
     f ++ fr"asc"
 
+  def max: Fragment =
+    fr"MAX(" ++ f ++ fr")"
 }
diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
index dbb5dc16..b997bb5e 100644
--- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
@@ -38,8 +38,10 @@ object RAttachment {
       fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}"
     ).update.run
 
-  def countOnItem(id: Ident): ConnectionIO[Int] =
-    selectCount(itemId, table, itemId.is(id)).query[Int].unique
+  def nextPosition(id: Ident): ConnectionIO[Int] =
+    for {
+      max <- selectSimple(position.max, table, itemId.is(id)).query[Option[Int]].unique
+    } yield max.map(_ + 1).getOrElse(0)
 
   def updateFileIdAndName(
       attachId: Ident,

From bcd13bed94ecd728351de504e807367bd71b0e1e Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 24 May 2020 15:07:25 +0200
Subject: [PATCH 15/15] Show information on what files to upload

---
 modules/webapp/src/main/elm/Comp/Dropzone.elm | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/modules/webapp/src/main/elm/Comp/Dropzone.elm b/modules/webapp/src/main/elm/Comp/Dropzone.elm
index b85ff1ae..67268f8b 100644
--- a/modules/webapp/src/main/elm/Comp/Dropzone.elm
+++ b/modules/webapp/src/main/elm/Comp/Dropzone.elm
@@ -136,6 +136,12 @@ view model =
             [ i [ class "folder open icon" ] []
             , text "Select ..."
             ]
+        , div [ class "ui center aligned text container" ]
+            [ span [ class "small-info" ]
+                [ text "Choose document files (pdf, docx, txt, html, …). "
+                , text "Archives (zip and eml) are extracted."
+                ]
+            ]
         ]