From c9de74fd912bd5504eda94843e9fa529115b5610 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Tue, 5 May 2020 22:48:08 +0200
Subject: [PATCH 01/14] Add imap settings

---
 .../scala/docspell/backend/ops/OMail.scala    |  97 +++++-
 .../src/main/resources/docspell-openapi.yml   | 134 +++++++-
 .../routes/MailSettingsRoutes.scala           | 112 ++++++-
 .../routes/NotifyDueItemsRoutes.scala         |   2 +-
 .../db/migration/mariadb/V1.5.0__userimap.sql |  14 +
 .../migration/postgresql/V1.5.0__userimap.sql |  14 +
 .../docspell/store/records/RUserImap.scala    | 201 ++++++++++++
 modules/webapp/src/main/elm/Api.elm           |  56 +++-
 .../src/main/elm/Comp/ImapSettingsForm.elm    | 226 ++++++++++++++
 .../src/main/elm/Comp/ImapSettingsManage.elm  | 288 ++++++++++++++++++
 .../src/main/elm/Comp/ImapSettingsTable.elm   |  74 +++++
 .../src/main/elm/Page/UserSettings/Data.elm   |   5 +
 .../src/main/elm/Page/UserSettings/Update.elm |  15 +
 .../src/main/elm/Page/UserSettings/View.elm   |  21 +-
 14 files changed, 1221 insertions(+), 38 deletions(-)
 create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.5.0__userimap.sql
 create mode 100644 modules/store/src/main/resources/db/migration/postgresql/V1.5.0__userimap.sql
 create mode 100644 modules/store/src/main/scala/docspell/store/records/RUserImap.scala
 create mode 100644 modules/webapp/src/main/elm/Comp/ImapSettingsForm.elm
 create mode 100644 modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm
 create mode 100644 modules/webapp/src/main/elm/Comp/ImapSettingsTable.elm

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OMail.scala b/modules/backend/src/main/scala/docspell/backend/ops/OMail.scala
index c334964e..4c4ae045 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OMail.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OMail.scala
@@ -12,19 +12,29 @@ import docspell.common._
 import docspell.store._
 import docspell.store.records._
 import docspell.store.queries.QMails
-import OMail.{ItemMail, Sent, SmtpSettings}
+import OMail.{ImapSettings, ItemMail, Sent, SmtpSettings}
 
 trait OMail[F[_]] {
 
-  def getSettings(accId: AccountId, nameQ: Option[String]): F[Vector[RUserEmail]]
+  def getSmtpSettings(accId: AccountId, nameQ: Option[String]): F[Vector[RUserEmail]]
 
-  def findSettings(accId: AccountId, name: Ident): OptionT[F, RUserEmail]
+  def findSmtpSettings(accId: AccountId, name: Ident): OptionT[F, RUserEmail]
 
-  def createSettings(accId: AccountId, data: SmtpSettings): F[AddResult]
+  def createSmtpSettings(accId: AccountId, data: SmtpSettings): F[AddResult]
 
-  def updateSettings(accId: AccountId, name: Ident, data: OMail.SmtpSettings): F[Int]
+  def updateSmtpSettings(accId: AccountId, name: Ident, data: OMail.SmtpSettings): F[Int]
 
-  def deleteSettings(accId: AccountId, name: Ident): F[Int]
+  def deleteSmtpSettings(accId: AccountId, name: Ident): F[Int]
+
+  def getImapSettings(accId: AccountId, nameQ: Option[String]): F[Vector[RUserImap]]
+
+  def findImapSettings(accId: AccountId, name: Ident): OptionT[F, RUserImap]
+
+  def createImapSettings(accId: AccountId, data: ImapSettings): F[AddResult]
+
+  def updateImapSettings(accId: AccountId, name: Ident, data: OMail.ImapSettings): F[Int]
+
+  def deleteImapSettings(accId: AccountId, name: Ident): F[Int]
 
   def sendMail(accId: AccountId, name: Ident, m: ItemMail): F[SendResult]
 
@@ -103,15 +113,41 @@ object OMail {
       )
   }
 
+  case class ImapSettings(
+      name: Ident,
+      imapHost: String,
+      imapPort: Option[Int],
+      imapUser: Option[String],
+      imapPassword: Option[Password],
+      imapSsl: SSLType,
+      imapCertCheck: Boolean
+  ) {
+
+    def toRecord(accId: AccountId) =
+      RUserImap.fromAccount(
+        accId,
+        name,
+        imapHost,
+        imapPort,
+        imapUser,
+        imapPassword,
+        imapSsl,
+        imapCertCheck
+      )
+  }
+
   def apply[F[_]: Effect](store: Store[F], emil: Emil[F]): Resource[F, OMail[F]] =
     Resource.pure[F, OMail[F]](new OMail[F] {
-      def getSettings(accId: AccountId, nameQ: Option[String]): F[Vector[RUserEmail]] =
+      def getSmtpSettings(
+          accId: AccountId,
+          nameQ: Option[String]
+      ): F[Vector[RUserEmail]] =
         store.transact(RUserEmail.findByAccount(accId, nameQ))
 
-      def findSettings(accId: AccountId, name: Ident): OptionT[F, RUserEmail] =
+      def findSmtpSettings(accId: AccountId, name: Ident): OptionT[F, RUserEmail] =
         OptionT(store.transact(RUserEmail.getByName(accId, name)))
 
-      def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
+      def createSmtpSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
         (for {
           ru <- OptionT(store.transact(s.toRecord(accId).value))
           ins    = RUserEmail.insert(ru)
@@ -119,7 +155,11 @@ object OMail {
           res <- OptionT.liftF(store.add(ins, exists))
         } yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
 
-      def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = {
+      def updateSmtpSettings(
+          accId: AccountId,
+          name: Ident,
+          data: SmtpSettings
+      ): F[Int] = {
         val op = for {
           um <- OptionT(RUserEmail.getByName(accId, name))
           ru <- data.toRecord(accId)
@@ -129,12 +169,43 @@ object OMail {
         store.transact(op.value).map(_.getOrElse(0))
       }
 
-      def deleteSettings(accId: AccountId, name: Ident): F[Int] =
+      def deleteSmtpSettings(accId: AccountId, name: Ident): F[Int] =
         store.transact(RUserEmail.delete(accId, name))
 
+      def getImapSettings(accId: AccountId, nameQ: Option[String]): F[Vector[RUserImap]] =
+        store.transact(RUserImap.findByAccount(accId, nameQ))
+
+      def findImapSettings(accId: AccountId, name: Ident): OptionT[F, RUserImap] =
+        OptionT(store.transact(RUserImap.getByName(accId, name)))
+
+      def createImapSettings(accId: AccountId, data: ImapSettings): F[AddResult] =
+        (for {
+          ru <- OptionT(store.transact(data.toRecord(accId).value))
+          ins    = RUserImap.insert(ru)
+          exists = RUserImap.exists(ru.uid, ru.name)
+          res <- OptionT.liftF(store.add(ins, exists))
+        } yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
+
+      def updateImapSettings(
+          accId: AccountId,
+          name: Ident,
+          data: OMail.ImapSettings
+      ): F[Int] = {
+        val op = for {
+          um <- OptionT(RUserImap.getByName(accId, name))
+          ru <- data.toRecord(accId)
+          n  <- OptionT.liftF(RUserImap.update(um.id, ru))
+        } yield n
+
+        store.transact(op.value).map(_.getOrElse(0))
+      }
+
+      def deleteImapSettings(accId: AccountId, name: Ident): F[Int] =
+        store.transact(RUserImap.delete(accId, name))
+
       def sendMail(accId: AccountId, name: Ident, m: ItemMail): F[SendResult] = {
 
-        val getSettings: OptionT[F, RUserEmail] =
+        val getSmtpSettings: OptionT[F, RUserEmail] =
           OptionT(store.transact(RUserEmail.getByName(accId, name)))
 
         def createMail(sett: RUserEmail): OptionT[F, Mail[F]] = {
@@ -198,7 +269,7 @@ object OMail {
         }
 
         (for {
-          mailCfg <- getSettings
+          mailCfg <- getSmtpSettings
           mail    <- createMail(mailCfg)
           mid     <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail))
           res     <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg)))
diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml
index 61ad220a..642a8bba 100644
--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -1414,7 +1414,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/BasicResult"
-  /sec/email/settings:
+  /sec/email/settings/smtp:
     get:
       tags: [ E-Mail ]
       summary: List email settings for current user.
@@ -1456,7 +1456,7 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/BasicResult"
-  /sec/email/settings/{name}:
+  /sec/email/settings/smtp/{name}:
     parameters:
       - $ref: "#/components/parameters/name"
     get:
@@ -1507,6 +1507,99 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/BasicResult"
+  /sec/email/settings/imap:
+    get:
+      tags: [ E-Mail ]
+      summary: List email settings for current user.
+      description: |
+        List all available e-mail settings for the current user.
+        E-Mail settings specify imap connections that can be used to
+        retrieve e-mails.
+
+        Multiple e-mail settings can be specified, they are
+        distinguished by their `name`. The query `q` parameter does a
+        simple substring search in the connection name.
+      security:
+        - authTokenHeader: []
+      parameters:
+        - $ref: "#/components/parameters/q"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ImapSettingsList"
+    post:
+      tags: [ E-Mail ]
+      summary: Create new email settings
+      description: |
+        Create new e-mail settings.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ImapSettings"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
+  /sec/email/settings/imap/{name}:
+    parameters:
+      - $ref: "#/components/parameters/name"
+    get:
+      tags: [ E-Mail ]
+      summary: Return specific email settings by name.
+      description: |
+        Return the stored e-mail settings for the given connection
+        name.
+      security:
+        - authTokenHeader: []
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ImapSettings"
+    put:
+      tags: [ E-Mail ]
+      summary: Change specific email settings.
+      description: |
+        Changes all settings for the connection with the given `name`.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ImapSettings"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
+    delete:
+      tags: [ E-Mail ]
+      summary: Delete e-mail settings.
+      description: |
+        Deletes the e-mail settings with the specified `name`.
+      security:
+        - authTokenHeader: []
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
 
   /sec/email/send/{name}/{id}:
     post:
@@ -1664,6 +1757,43 @@ paths:
 
 components:
   schemas:
+    ImapSettingsList:
+      description: |
+        A list of user email settings.
+      required:
+        - items
+      properties:
+        items:
+          type: array
+          items:
+            $ref: "#/components/schemas/ImapSettings"
+    ImapSettings:
+      description: |
+        IMAP settings for sending mail.
+      required:
+        - name
+        - imapHost
+        - from
+        - sslType
+        - ignoreCertificates
+      properties:
+        name:
+          type: string
+          format: ident
+        imapHost:
+          type: string
+        imapPort:
+          type: integer
+          format: int32
+        imapUser:
+          type: string
+        imapPassword:
+          type: string
+          format: password
+        sslType:
+          type: string
+        ignoreCertificates:
+          type: boolean
     CalEventCheckResult:
       description: |
         The result of checking a calendar event string.
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/MailSettingsRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/MailSettingsRoutes.scala
index 38a321a8..a3bf338f 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/MailSettingsRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/MailSettingsRoutes.scala
@@ -15,7 +15,7 @@ import docspell.backend.auth.AuthToken
 import docspell.backend.ops.OMail
 import docspell.common._
 import docspell.restapi.model._
-import docspell.store.records.RUserEmail
+import docspell.store.records.{RUserEmail, RUserImap}
 import docspell.restserver.conv.Conversions
 import docspell.restserver.http4s.QueryParam
 
@@ -26,25 +26,38 @@ object MailSettingsRoutes {
     import dsl._
 
     HttpRoutes.of {
-      case GET -> Root :? QueryParam.QueryOpt(q) =>
+      case GET -> Root / "smtp" :? QueryParam.QueryOpt(q) =>
         for {
-          list <- backend.mail.getSettings(user.account, q.map(_.q))
+          list <- backend.mail.getSmtpSettings(user.account, q.map(_.q))
           res = list.map(convert)
           resp <- Ok(EmailSettingsList(res.toList))
         } yield resp
 
-      case GET -> Root / Ident(name) =>
+      case GET -> Root / "imap" :? QueryParam.QueryOpt(q) =>
+        for {
+          list <- backend.mail.getImapSettings(user.account, q.map(_.q))
+          res = list.map(convert)
+          resp <- Ok(ImapSettingsList(res.toList))
+        } yield resp
+
+      case GET -> Root / "smtp" / Ident(name) =>
         (for {
-          ems  <- backend.mail.findSettings(user.account, name)
+          ems  <- backend.mail.findSmtpSettings(user.account, name)
           resp <- OptionT.liftF(Ok(convert(ems)))
         } yield resp).getOrElseF(NotFound())
 
-      case req @ POST -> Root =>
+      case GET -> Root / "imap" / Ident(name) =>
+        (for {
+          ems  <- backend.mail.findImapSettings(user.account, name)
+          resp <- OptionT.liftF(Ok(convert(ems)))
+        } yield resp).getOrElseF(NotFound())
+
+      case req @ POST -> Root / "smtp" =>
         (for {
           in <- OptionT.liftF(req.as[EmailSettings])
-          ru = makeSettings(in)
+          ru = makeSmtpSettings(in)
           up <- OptionT.liftF(
-            ru.traverse(r => backend.mail.createSettings(user.account, r))
+            ru.traverse(r => backend.mail.createSmtpSettings(user.account, r))
           )
           resp <- OptionT.liftF(
             Ok(
@@ -56,12 +69,29 @@ object MailSettingsRoutes {
           )
         } yield resp).getOrElseF(NotFound())
 
-      case req @ PUT -> Root / Ident(name) =>
+      case req @ POST -> Root / "imap" =>
+        (for {
+          in <- OptionT.liftF(req.as[ImapSettings])
+          ru = makeImapSettings(in)
+          up <- OptionT.liftF(
+            ru.traverse(r => backend.mail.createImapSettings(user.account, r))
+          )
+          resp <- OptionT.liftF(
+            Ok(
+              up.fold(
+                err => BasicResult(false, err),
+                ar => Conversions.basicResult(ar, "Mail settings stored.")
+              )
+            )
+          )
+        } yield resp).getOrElseF(NotFound())
+
+      case req @ PUT -> Root / "smtp" / Ident(name) =>
         (for {
           in <- OptionT.liftF(req.as[EmailSettings])
-          ru = makeSettings(in)
+          ru = makeSmtpSettings(in)
           up <- OptionT.liftF(
-            ru.traverse(r => backend.mail.updateSettings(user.account, name, r))
+            ru.traverse(r => backend.mail.updateSmtpSettings(user.account, name, r))
           )
           resp <- OptionT.liftF(
             Ok(
@@ -75,16 +105,43 @@ object MailSettingsRoutes {
           )
         } yield resp).getOrElseF(NotFound())
 
-      case DELETE -> Root / Ident(name) =>
+      case req @ PUT -> Root / "imap" / Ident(name) =>
+        (for {
+          in <- OptionT.liftF(req.as[ImapSettings])
+          ru = makeImapSettings(in)
+          up <- OptionT.liftF(
+            ru.traverse(r => backend.mail.updateImapSettings(user.account, name, r))
+          )
+          resp <- OptionT.liftF(
+            Ok(
+              up.fold(
+                err => BasicResult(false, err),
+                n =>
+                  if (n > 0) BasicResult(true, "Mail settings stored.")
+                  else BasicResult(false, "Mail settings could not be saved")
+              )
+            )
+          )
+        } yield resp).getOrElseF(NotFound())
+
+      case DELETE -> Root / "smtp" / Ident(name) =>
         for {
-          n <- backend.mail.deleteSettings(user.account, name)
+          n <- backend.mail.deleteSmtpSettings(user.account, name)
+          resp <- Ok(
+            if (n > 0) BasicResult(true, "Mail settings removed")
+            else BasicResult(false, "Mail settings could not be removed")
+          )
+        } yield resp
+
+      case DELETE -> Root / "imap" / Ident(name) =>
+        for {
+          n <- backend.mail.deleteImapSettings(user.account, name)
           resp <- Ok(
             if (n > 0) BasicResult(true, "Mail settings removed")
             else BasicResult(false, "Mail settings could not be removed")
           )
         } yield resp
     }
-
   }
 
   def convert(ru: RUserEmail): EmailSettings =
@@ -100,7 +157,18 @@ object MailSettingsRoutes {
       !ru.smtpCertCheck
     )
 
-  def makeSettings(ems: EmailSettings): Either[String, OMail.SmtpSettings] = {
+  def convert(ru: RUserImap): ImapSettings =
+    ImapSettings(
+      ru.name,
+      ru.imapHost,
+      ru.imapPort,
+      ru.imapUser,
+      ru.imapPassword,
+      ru.imapSsl.name,
+      !ru.imapCertCheck
+    )
+
+  def makeSmtpSettings(ems: EmailSettings): Either[String, OMail.SmtpSettings] = {
     def readMail(str: String): Either[String, MailAddress] =
       MailAddress.parse(str).left.map(err => s"E-Mail address '$str' invalid: $err")
 
@@ -122,6 +190,18 @@ object MailSettingsRoutes {
       from,
       repl
     )
-
   }
+
+  def makeImapSettings(ims: ImapSettings): Either[String, OMail.ImapSettings] =
+    for {
+      sslt <- SSLType.fromString(ims.sslType)
+    } yield OMail.ImapSettings(
+      ims.name,
+      ims.imapHost,
+      ims.imapPort,
+      ims.imapUser,
+      ims.imapPassword,
+      sslt,
+      !ims.ignoreCertificates
+    )
 }
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala
index 069346d4..88c7a67f 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala
@@ -91,7 +91,7 @@ object NotifyDueItemsRoutes {
       texc <- backend.tag.loadAll(task.args.tagsExclude)
       conn <-
         backend.mail
-          .getSettings(account, None)
+          .getSmtpSettings(account, None)
           .map(
             _.find(_.name == task.args.smtpConnection)
               .map(_.name)
diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.5.0__userimap.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.5.0__userimap.sql
new file mode 100644
index 00000000..2795b487
--- /dev/null
+++ b/modules/store/src/main/resources/db/migration/mariadb/V1.5.0__userimap.sql
@@ -0,0 +1,14 @@
+CREATE TABLE `userimap` (
+  `id` varchar(254) not null primary key,
+  `uid` varchar(254) not null,
+  `name` varchar(254) not null,
+  `imap_host` varchar(254) not null,
+  `imap_port` int,
+  `imap_user` varchar(254),
+  `imap_password` varchar(254),
+  `imap_ssl` varchar(254) not null,
+  `imap_certcheck` boolean not null,
+  `created` timestamp not null,
+  unique (`uid`, `name`),
+  foreign key (`uid`) references `user_`(`uid`)
+);
diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.5.0__userimap.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.5.0__userimap.sql
new file mode 100644
index 00000000..7b60396f
--- /dev/null
+++ b/modules/store/src/main/resources/db/migration/postgresql/V1.5.0__userimap.sql
@@ -0,0 +1,14 @@
+CREATE TABLE "userimap" (
+  "id" varchar(254) not null primary key,
+  "uid" varchar(254) not null,
+  "name" varchar(254) not null,
+  "imap_host" varchar(254) not null,
+  "imap_port" int,
+  "imap_user" varchar(254),
+  "imap_password" varchar(254),
+  "imap_ssl" varchar(254) not null,
+  "imap_certcheck" boolean not null,
+  "created" timestamp not null,
+  unique ("uid", "name"),
+  foreign key ("uid") references "user_"("uid")
+);
diff --git a/modules/store/src/main/scala/docspell/store/records/RUserImap.scala b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala
new file mode 100644
index 00000000..433fdd1a
--- /dev/null
+++ b/modules/store/src/main/scala/docspell/store/records/RUserImap.scala
@@ -0,0 +1,201 @@
+package docspell.store.records
+
+import doobie._
+import doobie.implicits._
+import cats.effect._
+import cats.implicits._
+import cats.data.OptionT
+import docspell.common._
+import docspell.store.impl.Column
+import docspell.store.impl.Implicits._
+import emil.{MailConfig, SSLType}
+
+case class RUserImap(
+    id: Ident,
+    uid: Ident,
+    name: Ident,
+    imapHost: String,
+    imapPort: Option[Int],
+    imapUser: Option[String],
+    imapPassword: Option[Password],
+    imapSsl: SSLType,
+    imapCertCheck: Boolean,
+    created: Timestamp
+) {
+
+  def toMailConfig: MailConfig = {
+    val port = imapPort.map(p => s":$p").getOrElse("")
+    MailConfig(
+      s"imap://$imapHost$port",
+      imapUser.getOrElse(""),
+      imapPassword.map(_.pass).getOrElse(""),
+      imapSsl,
+      !imapCertCheck
+    )
+  }
+}
+
+object RUserImap {
+
+  def apply[F[_]: Sync](
+      uid: Ident,
+      name: Ident,
+      imapHost: String,
+      imapPort: Option[Int],
+      imapUser: Option[String],
+      imapPassword: Option[Password],
+      imapSsl: SSLType,
+      imapCertCheck: Boolean
+  ): F[RUserImap] =
+    for {
+      now <- Timestamp.current[F]
+      id  <- Ident.randomId[F]
+    } yield RUserImap(
+      id,
+      uid,
+      name,
+      imapHost,
+      imapPort,
+      imapUser,
+      imapPassword,
+      imapSsl,
+      imapCertCheck,
+      now
+    )
+
+  def fromAccount(
+      accId: AccountId,
+      name: Ident,
+      imapHost: String,
+      imapPort: Option[Int],
+      imapUser: Option[String],
+      imapPassword: Option[Password],
+      imapSsl: SSLType,
+      imapCertCheck: Boolean
+  ): OptionT[ConnectionIO, RUserImap] =
+    for {
+      now  <- OptionT.liftF(Timestamp.current[ConnectionIO])
+      id   <- OptionT.liftF(Ident.randomId[ConnectionIO])
+      user <- OptionT(RUser.findByAccount(accId))
+    } yield RUserImap(
+      id,
+      user.uid,
+      name,
+      imapHost,
+      imapPort,
+      imapUser,
+      imapPassword,
+      imapSsl,
+      imapCertCheck,
+      now
+    )
+
+  val table = fr"userimap"
+
+  object Columns {
+    val id            = Column("id")
+    val uid           = Column("uid")
+    val name          = Column("name")
+    val imapHost      = Column("imap_host")
+    val imapPort      = Column("imap_port")
+    val imapUser      = Column("imap_user")
+    val imapPass      = Column("imap_password")
+    val imapSsl       = Column("imap_ssl")
+    val imapCertCheck = Column("imap_certcheck")
+    val created       = Column("created")
+
+    val all = List(
+      id,
+      uid,
+      name,
+      imapHost,
+      imapPort,
+      imapUser,
+      imapPass,
+      imapSsl,
+      imapCertCheck,
+      created
+    )
+  }
+
+  import Columns._
+
+  def insert(v: RUserImap): ConnectionIO[Int] =
+    insertRow(
+      table,
+      all,
+      sql"${v.id},${v.uid},${v.name},${v.imapHost},${v.imapPort},${v.imapUser},${v.imapPassword},${v.imapSsl},${v.imapCertCheck},${v.created}"
+    ).update.run
+
+  def update(eId: Ident, v: RUserImap): ConnectionIO[Int] =
+    updateRow(
+      table,
+      id.is(eId),
+      commas(
+        name.setTo(v.name),
+        imapHost.setTo(v.imapHost),
+        imapPort.setTo(v.imapPort),
+        imapUser.setTo(v.imapUser),
+        imapPass.setTo(v.imapPassword),
+        imapSsl.setTo(v.imapSsl),
+        imapCertCheck.setTo(v.imapCertCheck)
+      )
+    ).update.run
+
+  def findByUser(userId: Ident): ConnectionIO[Vector[RUserImap]] =
+    selectSimple(all, table, uid.is(userId)).query[RUserImap].to[Vector]
+
+  private def findByAccount0(
+      accId: AccountId,
+      nameQ: Option[String],
+      exact: Boolean
+  ): Query0[RUserImap] = {
+    val mUid   = uid.prefix("m")
+    val mName  = name.prefix("m")
+    val uId    = RUser.Columns.uid.prefix("u")
+    val uColl  = RUser.Columns.cid.prefix("u")
+    val uLogin = RUser.Columns.login.prefix("u")
+    val from   = table ++ fr"m INNER JOIN" ++ RUser.table ++ fr"u ON" ++ mUid.is(uId)
+    val cond = Seq(uColl.is(accId.collective), uLogin.is(accId.user)) ++ (nameQ match {
+      case Some(str) if exact  => Seq(mName.is(str))
+      case Some(str) if !exact => Seq(mName.lowerLike(s"%${str.toLowerCase}%"))
+      case None                => Seq.empty
+    })
+
+    (selectSimple(all.map(_.prefix("m")), from, and(cond)) ++ orderBy(mName.f))
+      .query[RUserImap]
+  }
+
+  def findByAccount(
+      accId: AccountId,
+      nameQ: Option[String]
+  ): ConnectionIO[Vector[RUserImap]] =
+    findByAccount0(accId, nameQ, false).to[Vector]
+
+  def getByName(accId: AccountId, name: Ident): ConnectionIO[Option[RUserImap]] =
+    findByAccount0(accId, Some(name.id), true).option
+
+  def delete(accId: AccountId, connName: Ident): ConnectionIO[Int] = {
+    val uId    = RUser.Columns.uid
+    val uColl  = RUser.Columns.cid
+    val uLogin = RUser.Columns.login
+    val cond   = Seq(uColl.is(accId.collective), uLogin.is(accId.user))
+
+    deleteFrom(
+      table,
+      fr"uid in (" ++ selectSimple(Seq(uId), RUser.table, and(cond)) ++ fr") AND" ++ name
+        .is(
+          connName
+        )
+    ).update.run
+  }
+
+  def exists(accId: AccountId, name: Ident): ConnectionIO[Boolean] =
+    getByName(accId, name).map(_.isDefined)
+
+  def exists(userId: Ident, connName: Ident): ConnectionIO[Boolean] =
+    selectCount(id, table, and(uid.is(userId), name.is(connName)))
+      .query[Int]
+      .unique
+      .map(_ > 0)
+}
diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm
index ec4581ab..e1e74060 100644
--- a/modules/webapp/src/main/elm/Api.elm
+++ b/modules/webapp/src/main/elm/Api.elm
@@ -2,9 +2,11 @@ module Api exposing
     ( cancelJob
     , changePassword
     , checkCalEvent
+    , createImapSettings
     , createMailSettings
     , deleteAttachment
     , deleteEquip
+    , deleteImapSettings
     , deleteItem
     , deleteMailSettings
     , deleteOrg
@@ -17,6 +19,7 @@ module Api exposing
     , getCollectiveSettings
     , getContacts
     , getEquipments
+    , getImapSettings
     , getInsights
     , getItemProposals
     , getJobQueueState
@@ -81,6 +84,8 @@ import Api.Model.EmailSettingsList exposing (EmailSettingsList)
 import Api.Model.Equipment exposing (Equipment)
 import Api.Model.EquipmentList exposing (EquipmentList)
 import Api.Model.GenInvite exposing (GenInvite)
+import Api.Model.ImapSettings exposing (ImapSettings)
+import Api.Model.ImapSettingsList exposing (ImapSettingsList)
 import Api.Model.InviteResult exposing (InviteResult)
 import Api.Model.ItemDetail exposing (ItemDetail)
 import Api.Model.ItemInsights exposing (ItemInsights)
@@ -259,7 +264,16 @@ sendMail flags opts receive =
 deleteMailSettings : Flags -> String -> (Result Http.Error BasicResult -> msg) -> Cmd msg
 deleteMailSettings flags name receive =
     Http2.authDelete
-        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/" ++ name
+        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/smtp/" ++ name
+        , account = getAccount flags
+        , expect = Http.expectJson receive Api.Model.BasicResult.decoder
+        }
+
+
+deleteImapSettings : Flags -> String -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+deleteImapSettings flags name receive =
+    Http2.authDelete
+        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/imap/" ++ name
         , account = getAccount flags
         , expect = Http.expectJson receive Api.Model.BasicResult.decoder
         }
@@ -268,12 +282,21 @@ deleteMailSettings flags name receive =
 getMailSettings : Flags -> String -> (Result Http.Error EmailSettingsList -> msg) -> Cmd msg
 getMailSettings flags query receive =
     Http2.authGet
-        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings?q=" ++ Url.percentEncode query
+        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/smtp?q=" ++ Url.percentEncode query
         , account = getAccount flags
         , expect = Http.expectJson receive Api.Model.EmailSettingsList.decoder
         }
 
 
+getImapSettings : Flags -> String -> (Result Http.Error ImapSettingsList -> msg) -> Cmd msg
+getImapSettings flags query receive =
+    Http2.authGet
+        { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/imap?q=" ++ Url.percentEncode query
+        , account = getAccount flags
+        , expect = Http.expectJson receive Api.Model.ImapSettingsList.decoder
+        }
+
+
 createMailSettings :
     Flags
     -> Maybe String
@@ -284,7 +307,7 @@ createMailSettings flags mname ems receive =
     case mname of
         Just en ->
             Http2.authPut
-                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/" ++ en
+                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/smtp/" ++ en
                 , account = getAccount flags
                 , body = Http.jsonBody (Api.Model.EmailSettings.encode ems)
                 , expect = Http.expectJson receive Api.Model.BasicResult.decoder
@@ -292,13 +315,38 @@ createMailSettings flags mname ems receive =
 
         Nothing ->
             Http2.authPost
-                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings"
+                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/smtp"
                 , account = getAccount flags
                 , body = Http.jsonBody (Api.Model.EmailSettings.encode ems)
                 , expect = Http.expectJson receive Api.Model.BasicResult.decoder
                 }
 
 
+createImapSettings :
+    Flags
+    -> Maybe String
+    -> ImapSettings
+    -> (Result Http.Error BasicResult -> msg)
+    -> Cmd msg
+createImapSettings flags mname ems receive =
+    case mname of
+        Just en ->
+            Http2.authPut
+                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/imap/" ++ en
+                , account = getAccount flags
+                , body = Http.jsonBody (Api.Model.ImapSettings.encode ems)
+                , expect = Http.expectJson receive Api.Model.BasicResult.decoder
+                }
+
+        Nothing ->
+            Http2.authPost
+                { url = flags.config.baseUrl ++ "/api/v1/sec/email/settings/imap"
+                , account = getAccount flags
+                , body = Http.jsonBody (Api.Model.ImapSettings.encode ems)
+                , expect = Http.expectJson receive Api.Model.BasicResult.decoder
+                }
+
+
 
 --- Upload
 
diff --git a/modules/webapp/src/main/elm/Comp/ImapSettingsForm.elm b/modules/webapp/src/main/elm/Comp/ImapSettingsForm.elm
new file mode 100644
index 00000000..f2d58420
--- /dev/null
+++ b/modules/webapp/src/main/elm/Comp/ImapSettingsForm.elm
@@ -0,0 +1,226 @@
+module Comp.ImapSettingsForm exposing
+    ( Model
+    , Msg
+    , emptyModel
+    , getSettings
+    , init
+    , isValid
+    , update
+    , view
+    )
+
+import Api.Model.ImapSettings exposing (ImapSettings)
+import Comp.Dropdown
+import Comp.IntField
+import Comp.PasswordInput
+import Data.SSLType exposing (SSLType)
+import Html exposing (..)
+import Html.Attributes exposing (..)
+import Html.Events exposing (onCheck, onInput)
+import Util.Maybe
+
+
+type alias Model =
+    { settings : ImapSettings
+    , name : String
+    , host : String
+    , portField : Comp.IntField.Model
+    , portNum : Maybe Int
+    , user : Maybe String
+    , passField : Comp.PasswordInput.Model
+    , password : Maybe String
+    , sslType : Comp.Dropdown.Model SSLType
+    , ignoreCertificates : Bool
+    }
+
+
+emptyModel : Model
+emptyModel =
+    { settings = Api.Model.ImapSettings.empty
+    , name = ""
+    , host = ""
+    , portField = Comp.IntField.init (Just 0) Nothing True "IMAP Port"
+    , portNum = Nothing
+    , user = Nothing
+    , passField = Comp.PasswordInput.init
+    , password = Nothing
+    , sslType =
+        Comp.Dropdown.makeSingleList
+            { makeOption = \s -> { value = Data.SSLType.toString s, text = Data.SSLType.label s }
+            , placeholder = ""
+            , options = Data.SSLType.all
+            , selected = Just Data.SSLType.None
+            }
+    , ignoreCertificates = False
+    }
+
+
+init : ImapSettings -> Model
+init ems =
+    { settings = ems
+    , name = ems.name
+    , host = ems.imapHost
+    , portField = Comp.IntField.init (Just 0) Nothing True "IMAP Port"
+    , portNum = ems.imapPort
+    , user = ems.imapUser
+    , passField = Comp.PasswordInput.init
+    , password = ems.imapPassword
+    , sslType =
+        Comp.Dropdown.makeSingleList
+            { makeOption = \s -> { value = Data.SSLType.toString s, text = Data.SSLType.label s }
+            , placeholder = ""
+            , options = Data.SSLType.all
+            , selected =
+                Data.SSLType.fromString ems.sslType
+                    |> Maybe.withDefault Data.SSLType.None
+                    |> Just
+            }
+    , ignoreCertificates = ems.ignoreCertificates
+    }
+
+
+getSettings : Model -> ( Maybe String, ImapSettings )
+getSettings model =
+    ( Util.Maybe.fromString model.settings.name
+    , { name = model.name
+      , imapHost = model.host
+      , imapUser = model.user
+      , imapPort = model.portNum
+      , imapPassword = model.password
+      , sslType =
+            Comp.Dropdown.getSelected model.sslType
+                |> List.head
+                |> Maybe.withDefault Data.SSLType.None
+                |> Data.SSLType.toString
+      , ignoreCertificates = model.ignoreCertificates
+      }
+    )
+
+
+type Msg
+    = SetName String
+    | SetHost String
+    | PortMsg Comp.IntField.Msg
+    | SetUser String
+    | PassMsg Comp.PasswordInput.Msg
+    | SSLTypeMsg (Comp.Dropdown.Msg SSLType)
+    | ToggleCheckCert
+
+
+isValid : Model -> Bool
+isValid model =
+    model.host /= "" && model.name /= ""
+
+
+update : Msg -> Model -> ( Model, Cmd Msg )
+update msg model =
+    case msg of
+        SetName str ->
+            ( { model | name = str }, Cmd.none )
+
+        SetHost str ->
+            ( { model | host = str }, Cmd.none )
+
+        PortMsg m ->
+            let
+                ( pm, val ) =
+                    Comp.IntField.update m model.portField
+            in
+            ( { model | portField = pm, portNum = val }, Cmd.none )
+
+        SetUser str ->
+            ( { model | user = Util.Maybe.fromString str }, Cmd.none )
+
+        PassMsg m ->
+            let
+                ( pm, val ) =
+                    Comp.PasswordInput.update m model.passField
+            in
+            ( { model | passField = pm, password = val }, Cmd.none )
+
+        SSLTypeMsg m ->
+            let
+                ( sm, sc ) =
+                    Comp.Dropdown.update m model.sslType
+            in
+            ( { model | sslType = sm }, Cmd.map SSLTypeMsg sc )
+
+        ToggleCheckCert ->
+            ( { model | ignoreCertificates = not model.ignoreCertificates }, Cmd.none )
+
+
+view : Model -> Html Msg
+view model =
+    div
+        [ classList
+            [ ( "ui form", True )
+            , ( "error", not (isValid model) )
+            , ( "success", isValid model )
+            ]
+        ]
+        [ div [ class "required field" ]
+            [ label [] [ text "Name" ]
+            , input
+                [ type_ "text"
+                , value model.name
+                , onInput SetName
+                , placeholder "Connection name, e.g. 'gmail.com'"
+                ]
+                []
+            , div [ class "ui info message" ]
+                [ text "The connection name must not contain whitespace or special characters."
+                ]
+            ]
+        , div [ class "fields" ]
+            [ div [ class "thirteen wide required field" ]
+                [ label [] [ text "IMAP Host" ]
+                , input
+                    [ type_ "text"
+                    , placeholder "IMAP host name, e.g. 'mail.gmail.com'"
+                    , value model.host
+                    , onInput SetHost
+                    ]
+                    []
+                ]
+            , Html.map PortMsg
+                (Comp.IntField.view model.portNum
+                    "three wide field"
+                    model.portField
+                )
+            ]
+        , div [ class "two fields" ]
+            [ div [ class "field" ]
+                [ label [] [ text "IMAP User" ]
+                , input
+                    [ type_ "text"
+                    , placeholder "IMAP Username, e.g. 'your.name@gmail.com'"
+                    , Maybe.withDefault "" model.user |> value
+                    , onInput SetUser
+                    ]
+                    []
+                ]
+            , div [ class "field" ]
+                [ label [] [ text "IMAP Password" ]
+                , Html.map PassMsg (Comp.PasswordInput.view model.password model.passField)
+                ]
+            ]
+        , div [ class "two fields" ]
+            [ div [ class "inline field" ]
+                [ div [ class "ui checkbox" ]
+                    [ input
+                        [ type_ "checkbox"
+                        , checked model.ignoreCertificates
+                        , onCheck (\_ -> ToggleCheckCert)
+                        ]
+                        []
+                    , label [] [ text "Ignore certificate check" ]
+                    ]
+                ]
+            ]
+        , div [ class "two fields" ]
+            [ div [ class "field" ]
+                [ label [] [ text "SSL" ]
+                , Html.map SSLTypeMsg (Comp.Dropdown.view model.sslType)
+                ]
+            ]
+        ]
diff --git a/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm b/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm
new file mode 100644
index 00000000..5db98fb6
--- /dev/null
+++ b/modules/webapp/src/main/elm/Comp/ImapSettingsManage.elm
@@ -0,0 +1,288 @@
+module Comp.ImapSettingsManage exposing
+    ( Model
+    , Msg
+    , emptyModel
+    , init
+    , update
+    , view
+    )
+
+import Api
+import Api.Model.BasicResult exposing (BasicResult)
+import Api.Model.ImapSettings
+import Api.Model.ImapSettingsList exposing (ImapSettingsList)
+import Comp.ImapSettingsForm
+import Comp.ImapSettingsTable
+import Comp.YesNoDimmer
+import Data.Flags exposing (Flags)
+import Html exposing (..)
+import Html.Attributes exposing (..)
+import Html.Events exposing (onClick, onInput)
+import Http
+import Util.Http
+
+
+type alias Model =
+    { tableModel : Comp.ImapSettingsTable.Model
+    , formModel : Comp.ImapSettingsForm.Model
+    , viewMode : ViewMode
+    , formError : Maybe String
+    , loading : Bool
+    , query : String
+    , deleteConfirm : Comp.YesNoDimmer.Model
+    }
+
+
+emptyModel : Model
+emptyModel =
+    { tableModel = Comp.ImapSettingsTable.emptyModel
+    , formModel = Comp.ImapSettingsForm.emptyModel
+    , viewMode = Table
+    , formError = Nothing
+    , loading = False
+    , query = ""
+    , deleteConfirm = Comp.YesNoDimmer.emptyModel
+    }
+
+
+init : Flags -> ( Model, Cmd Msg )
+init flags =
+    ( emptyModel, Api.getImapSettings flags "" MailSettingsResp )
+
+
+type ViewMode
+    = Table
+    | Form
+
+
+type Msg
+    = TableMsg Comp.ImapSettingsTable.Msg
+    | FormMsg Comp.ImapSettingsForm.Msg
+    | SetQuery String
+    | InitNew
+    | YesNoMsg Comp.YesNoDimmer.Msg
+    | RequestDelete
+    | SetViewMode ViewMode
+    | Submit
+    | SubmitResp (Result Http.Error BasicResult)
+    | LoadSettings
+    | MailSettingsResp (Result Http.Error ImapSettingsList)
+
+
+update : Flags -> Msg -> Model -> ( Model, Cmd Msg )
+update flags msg model =
+    case msg of
+        InitNew ->
+            let
+                ems =
+                    Api.Model.ImapSettings.empty
+
+                nm =
+                    { model
+                        | viewMode = Form
+                        , formError = Nothing
+                        , formModel = Comp.ImapSettingsForm.init ems
+                    }
+            in
+            ( nm, Cmd.none )
+
+        TableMsg m ->
+            let
+                ( tm, tc ) =
+                    Comp.ImapSettingsTable.update m model.tableModel
+
+                m2 =
+                    { model
+                        | tableModel = tm
+                        , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table
+                        , formError =
+                            if tm.selected /= Nothing then
+                                Nothing
+
+                            else
+                                model.formError
+                        , formModel =
+                            case tm.selected of
+                                Just ems ->
+                                    Comp.ImapSettingsForm.init ems
+
+                                Nothing ->
+                                    model.formModel
+                    }
+            in
+            ( m2, Cmd.map TableMsg tc )
+
+        FormMsg m ->
+            let
+                ( fm, fc ) =
+                    Comp.ImapSettingsForm.update m model.formModel
+            in
+            ( { model | formModel = fm }, Cmd.map FormMsg fc )
+
+        SetQuery str ->
+            let
+                m =
+                    { model | query = str }
+            in
+            ( m, Api.getImapSettings flags str MailSettingsResp )
+
+        YesNoMsg m ->
+            let
+                ( dm, flag ) =
+                    Comp.YesNoDimmer.update m model.deleteConfirm
+
+                ( mid, _ ) =
+                    Comp.ImapSettingsForm.getSettings model.formModel
+
+                cmd =
+                    case ( flag, mid ) of
+                        ( True, Just name ) ->
+                            Api.deleteImapSettings flags name SubmitResp
+
+                        _ ->
+                            Cmd.none
+            in
+            ( { model | deleteConfirm = dm }, cmd )
+
+        RequestDelete ->
+            update flags (YesNoMsg Comp.YesNoDimmer.activate) model
+
+        SetViewMode m ->
+            ( { model | viewMode = m }, Cmd.none )
+
+        Submit ->
+            let
+                ( mid, ems ) =
+                    Comp.ImapSettingsForm.getSettings model.formModel
+
+                valid =
+                    Comp.ImapSettingsForm.isValid model.formModel
+            in
+            if valid then
+                ( { model | loading = True }, Api.createImapSettings flags mid ems SubmitResp )
+
+            else
+                ( { model | formError = Just "Please fill required fields." }, Cmd.none )
+
+        LoadSettings ->
+            ( { model | loading = True }, Api.getImapSettings flags model.query MailSettingsResp )
+
+        SubmitResp (Ok res) ->
+            if res.success then
+                let
+                    ( m2, c2 ) =
+                        update flags (SetViewMode Table) model
+
+                    ( m3, c3 ) =
+                        update flags LoadSettings m2
+                in
+                ( { m3 | loading = False }, Cmd.batch [ c2, c3 ] )
+
+            else
+                ( { model | formError = Just res.message, loading = False }, Cmd.none )
+
+        SubmitResp (Err err) ->
+            ( { model | formError = Just (Util.Http.errorToString err), loading = False }, Cmd.none )
+
+        MailSettingsResp (Ok ems) ->
+            let
+                m2 =
+                    { model
+                        | viewMode = Table
+                        , loading = False
+                        , tableModel = Comp.ImapSettingsTable.init ems.items
+                    }
+            in
+            ( m2, Cmd.none )
+
+        MailSettingsResp (Err _) ->
+            ( { model | loading = False }, Cmd.none )
+
+
+view : Model -> Html Msg
+view model =
+    case model.viewMode of
+        Table ->
+            viewTable model
+
+        Form ->
+            viewForm model
+
+
+viewTable : Model -> Html Msg
+viewTable model =
+    div []
+        [ div [ class "ui secondary menu" ]
+            [ div [ class "horizontally fitted item" ]
+                [ div [ class "ui icon input" ]
+                    [ input
+                        [ type_ "text"
+                        , onInput SetQuery
+                        , value model.query
+                        , placeholder "Search…"
+                        ]
+                        []
+                    , i [ class "ui search icon" ]
+                        []
+                    ]
+                ]
+            , div [ class "right menu" ]
+                [ div [ class "item" ]
+                    [ a
+                        [ class "ui primary button"
+                        , href "#"
+                        , onClick InitNew
+                        ]
+                        [ i [ class "plus icon" ] []
+                        , text "New Settings"
+                        ]
+                    ]
+                ]
+            ]
+        , Html.map TableMsg (Comp.ImapSettingsTable.view model.tableModel)
+        ]
+
+
+viewForm : Model -> Html Msg
+viewForm model =
+    div [ class "ui segment" ]
+        [ Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm)
+        , Html.map FormMsg (Comp.ImapSettingsForm.view model.formModel)
+        , div
+            [ classList
+                [ ( "ui error message", True )
+                , ( "invisible", model.formError == Nothing )
+                ]
+            ]
+            [ Maybe.withDefault "" model.formError |> text
+            ]
+        , div [ class "ui divider" ] []
+        , button
+            [ class "ui primary button"
+            , onClick Submit
+            , href "#"
+            ]
+            [ text "Submit"
+            ]
+        , a
+            [ class "ui secondary button"
+            , onClick (SetViewMode Table)
+            , href ""
+            ]
+            [ text "Cancel"
+            ]
+        , if model.formModel.settings.name /= "" then
+            a [ class "ui right floated red button", href "", onClick RequestDelete ]
+                [ text "Delete" ]
+
+          else
+            span [] []
+        , div
+            [ classList
+                [ ( "ui dimmer", True )
+                , ( "active", model.loading )
+                ]
+            ]
+            [ div [ class "ui loader" ] []
+            ]
+        ]
diff --git a/modules/webapp/src/main/elm/Comp/ImapSettingsTable.elm b/modules/webapp/src/main/elm/Comp/ImapSettingsTable.elm
new file mode 100644
index 00000000..a69bdc1e
--- /dev/null
+++ b/modules/webapp/src/main/elm/Comp/ImapSettingsTable.elm
@@ -0,0 +1,74 @@
+module Comp.ImapSettingsTable exposing
+    ( Model
+    , Msg
+    , emptyModel
+    , init
+    , update
+    , view
+    )
+
+import Api.Model.ImapSettings exposing (ImapSettings)
+import Html exposing (..)
+import Html.Attributes exposing (..)
+import Html.Events exposing (onClick)
+
+
+type alias Model =
+    { emailSettings : List ImapSettings
+    , selected : Maybe ImapSettings
+    }
+
+
+emptyModel : Model
+emptyModel =
+    init []
+
+
+init : List ImapSettings -> Model
+init ems =
+    { emailSettings = ems
+    , selected = Nothing
+    }
+
+
+type Msg
+    = Select ImapSettings
+
+
+update : Msg -> Model -> ( Model, Cmd Msg )
+update msg model =
+    case msg of
+        Select ems ->
+            ( { model | selected = Just ems }, Cmd.none )
+
+
+view : Model -> Html Msg
+view model =
+    table [ class "ui selectable pointer table" ]
+        [ thead []
+            [ th [ class "collapsible" ] [ text "Name" ]
+            , th [] [ text "Host/Port" ]
+            ]
+        , tbody []
+            (List.map (renderLine model) model.emailSettings)
+        ]
+
+
+renderLine : Model -> ImapSettings -> Html Msg
+renderLine model ems =
+    let
+        hostport =
+            case ems.imapPort of
+                Just p ->
+                    ems.imapHost ++ ":" ++ String.fromInt p
+
+                Nothing ->
+                    ems.imapHost
+    in
+    tr
+        [ classList [ ( "active", model.selected == Just ems ) ]
+        , onClick (Select ems)
+        ]
+        [ td [ class "collapsible" ] [ text ems.name ]
+        , td [] [ text hostport ]
+        ]
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Data.elm b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
index e8333c08..56276a09 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
@@ -7,6 +7,7 @@ module Page.UserSettings.Data exposing
 
 import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
+import Comp.ImapSettingsManage
 import Comp.NotificationForm
 import Data.Flags exposing (Flags)
 
@@ -15,6 +16,7 @@ type alias Model =
     { currentTab : Maybe Tab
     , changePassModel : Comp.ChangePasswordForm.Model
     , emailSettingsModel : Comp.EmailSettingsManage.Model
+    , imapSettingsModel : Comp.ImapSettingsManage.Model
     , notificationModel : Comp.NotificationForm.Model
     }
 
@@ -24,6 +26,7 @@ emptyModel flags =
     { currentTab = Nothing
     , changePassModel = Comp.ChangePasswordForm.emptyModel
     , emailSettingsModel = Comp.EmailSettingsManage.emptyModel
+    , imapSettingsModel = Comp.ImapSettingsManage.emptyModel
     , notificationModel = Tuple.first (Comp.NotificationForm.init flags)
     }
 
@@ -31,6 +34,7 @@ emptyModel flags =
 type Tab
     = ChangePassTab
     | EmailSettingsTab
+    | ImapSettingsTab
     | NotificationTab
 
 
@@ -39,3 +43,4 @@ type Msg
     | ChangePassMsg Comp.ChangePasswordForm.Msg
     | EmailSettingsMsg Comp.EmailSettingsManage.Msg
     | NotificationMsg Comp.NotificationForm.Msg
+    | ImapSettingsMsg Comp.ImapSettingsManage.Msg
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Update.elm b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
index 8dcf2f08..1a56cac1 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
@@ -2,6 +2,7 @@ module Page.UserSettings.Update exposing (update)
 
 import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
+import Comp.ImapSettingsManage
 import Comp.NotificationForm
 import Data.Flags exposing (Flags)
 import Page.UserSettings.Data exposing (..)
@@ -24,6 +25,13 @@ update flags msg model =
                             in
                             ( { m | emailSettingsModel = em }, Cmd.map EmailSettingsMsg c )
 
+                        ImapSettingsTab ->
+                            let
+                                ( em, c ) =
+                                    Comp.ImapSettingsManage.init flags
+                            in
+                            ( { m | imapSettingsModel = em }, Cmd.map ImapSettingsMsg c )
+
                         ChangePassTab ->
                             ( m, Cmd.none )
 
@@ -51,6 +59,13 @@ update flags msg model =
             in
             ( { model | emailSettingsModel = m2 }, Cmd.map EmailSettingsMsg c2 )
 
+        ImapSettingsMsg m ->
+            let
+                ( m2, c2 ) =
+                    Comp.ImapSettingsManage.update flags m model.imapSettingsModel
+            in
+            ( { model | imapSettingsModel = m2 }, Cmd.map ImapSettingsMsg c2 )
+
         NotificationMsg lm ->
             let
                 ( m2, c2 ) =
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
index 662dc425..bf02ba82 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/View.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
@@ -2,6 +2,7 @@ module Page.UserSettings.View exposing (view)
 
 import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
+import Comp.ImapSettingsManage
 import Comp.NotificationForm
 import Html exposing (..)
 import Html.Attributes exposing (..)
@@ -20,7 +21,8 @@ view model =
             , div [ class "ui attached fluid segment" ]
                 [ div [ class "ui fluid vertical secondary menu" ]
                     [ makeTab model ChangePassTab "Change Password" "user secret icon"
-                    , makeTab model EmailSettingsTab "E-Mail Settings" "mail icon"
+                    , makeTab model EmailSettingsTab "E-Mail Settings (SMTP)" "mail icon"
+                    , makeTab model ImapSettingsTab "E-Mail Settings (IMAP)" "mail icon"
                     , makeTab model NotificationTab "Notifications" "bullhorn icon"
                     ]
                 ]
@@ -37,6 +39,9 @@ view model =
                     Just NotificationTab ->
                         viewNotificationForm model
 
+                    Just ImapSettingsTab ->
+                        viewImapSettings model
+
                     Nothing ->
                         []
                 )
@@ -61,13 +66,25 @@ viewEmailSettings model =
     [ h2 [ class "ui header" ]
         [ i [ class "mail icon" ] []
         , div [ class "content" ]
-            [ text "E-Mail Settings"
+            [ text "E-Mail Settings (Smtp)"
             ]
         ]
     , Html.map EmailSettingsMsg (Comp.EmailSettingsManage.view model.emailSettingsModel)
     ]
 
 
+viewImapSettings : Model -> List (Html Msg)
+viewImapSettings model =
+    [ h2 [ class "ui header" ]
+        [ i [ class "mail icon" ] []
+        , div [ class "content" ]
+            [ text "E-Mail Settings (Imap)"
+            ]
+        ]
+    , Html.map ImapSettingsMsg (Comp.ImapSettingsManage.view model.imapSettingsModel)
+    ]
+
+
 viewChangePassword : Model -> List (Html Msg)
 viewChangePassword model =
     [ h2 [ class "ui header" ]

From 5d5311913c4956ba08412893cb44f9f4d86f39a3 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Sun, 10 May 2020 20:30:24 +0200
Subject: [PATCH 02/14] Add ScanMailboxArgs

---
 .../main/scala/docspell/common/Duration.scala |  9 +++-
 .../docspell/common/ScanMailboxArgs.scala     | 42 +++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala

diff --git a/modules/common/src/main/scala/docspell/common/Duration.scala b/modules/common/src/main/scala/docspell/common/Duration.scala
index 9c5cdd20..1f837196 100644
--- a/modules/common/src/main/scala/docspell/common/Duration.scala
+++ b/modules/common/src/main/scala/docspell/common/Duration.scala
@@ -4,7 +4,7 @@ import cats.implicits._
 import scala.concurrent.duration.{FiniteDuration, Duration => SDur}
 import java.time.{Duration => JDur}
 import java.util.concurrent.TimeUnit
-
+import io.circe._
 import cats.effect.Sync
 
 case class Duration(nanos: Long) {
@@ -54,4 +54,11 @@ object Duration {
       now <- Timestamp.current[F]
       end = Timestamp.current[F]
     } yield end.map(e => Duration.millis(e.toMillis - now.toMillis))
+
+
+  implicit val jsonEncoder: Encoder[Duration] =
+    Encoder.encodeLong.contramap(_.millis)
+
+  implicit val jsonDecoder: Decoder[Duration] =
+    Decoder.decodeLong.map(Duration.millis)
 }
diff --git a/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala
new file mode 100644
index 00000000..f84e30ea
--- /dev/null
+++ b/modules/common/src/main/scala/docspell/common/ScanMailboxArgs.scala
@@ -0,0 +1,42 @@
+package docspell.common
+
+import io.circe._, io.circe.generic.semiauto._
+import docspell.common.syntax.all._
+
+/** Arguments to the poll-mailbox task.
+  *
+  * This tasks queries user mailboxes and pushes found mails into
+  * docspell for processing.
+  *
+  * If the structure changes, there must be some database migration to
+  * update or remove the json data of the corresponding task.
+  */
+case class ScanMailboxArgs(
+    // the docspell user account
+    account: AccountId,
+    // the configured imap connection
+    imapConnection: Ident,
+    // what folders to search
+    folders: List[String],
+    // only select mails received since then
+    receivedSince: Option[Duration],
+    // move submitted mails to another folder
+    targetFolder: Option[String],
+    // delete the after submitting (only if targetFolder is None)
+    deleteMail: Boolean,
+    // set the direction when submitting
+    direction: Option[Direction]
+)
+
+object ScanMailboxArgs {
+
+  val taskName = Ident.unsafe("scan-mailbox")
+
+  implicit val jsonEncoder: Encoder[ScanMailboxArgs] =
+    deriveEncoder[ScanMailboxArgs]
+  implicit val jsonDecoder: Decoder[ScanMailboxArgs] =
+    deriveDecoder[ScanMailboxArgs]
+
+  def parse(str: String): Either[Throwable, ScanMailboxArgs] =
+    str.parseJsonAs[ScanMailboxArgs]
+}

From ebe65c76248ebc4cd4735bda270957805dc5d525 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Mon, 18 May 2020 08:56:38 +0200
Subject: [PATCH 03/14] Change menu title in user settings

---
 modules/webapp/src/main/elm/Page/UserSettings/View.elm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
index bf02ba82..4997c5ee 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/View.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
@@ -16,14 +16,14 @@ view model =
     div [ class "usersetting-page ui padded grid" ]
         [ div [ class "sixteen wide mobile four wide tablet four wide computer column" ]
             [ h4 [ class "ui top attached ablue-comp header" ]
-                [ text "User"
+                [ text "User Settings"
                 ]
             , div [ class "ui attached fluid segment" ]
                 [ div [ class "ui fluid vertical secondary menu" ]
                     [ makeTab model ChangePassTab "Change Password" "user secret icon"
                     , makeTab model EmailSettingsTab "E-Mail Settings (SMTP)" "mail icon"
                     , makeTab model ImapSettingsTab "E-Mail Settings (IMAP)" "mail icon"
-                    , makeTab model NotificationTab "Notifications" "bullhorn icon"
+                    , makeTab model NotificationTab "Notification Task" "bullhorn icon"
                     ]
                 ]
             ]

From a4be63fd77b52e923e1fec076639f8ee56472109 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Mon, 18 May 2020 09:12:17 +0200
Subject: [PATCH 04/14] Add stub for scan-mailbox task

---
 .../scala/docspell/joex/JoexAppImpl.scala     | 11 +++-
 .../docspell/joex/notify/MailContext.scala    |  1 +
 .../joex/scanmailbox/ScanMailboxTask.scala    | 55 +++++++++++++++++++
 3 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
index e7293e16..ac6ff7c7 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
@@ -6,6 +6,7 @@ import emil.javamail._
 import docspell.common._
 import docspell.joex.hk._
 import docspell.joex.notify._
+import docspell.joex.scanmailbox._
 import docspell.joex.process.ItemHandler
 import docspell.joex.scheduler._
 import docspell.joexapi.client.JoexClient
@@ -67,6 +68,7 @@ object JoexAppImpl {
       queue   <- JobQueue(store)
       pstore  <- PeriodicTaskStore.create(store)
       nodeOps <- ONode(store)
+      javaEmil = JavaMailEmil(blocker)
       sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
         .withQueue(queue)
         .withTask(
@@ -79,10 +81,17 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             NotifyDueItemsArgs.taskName,
-            NotifyDueItemsTask[F](cfg.sendMail, JavaMailEmil(blocker)),
+            NotifyDueItemsTask[F](cfg.sendMail, javaEmil),
             NotifyDueItemsTask.onCancel[F]
           )
         )
+        .withTask(
+          JobTask.json(
+            ScanMailboxArgs.taskName,
+            ScanMailboxTask[F](javaEmil),
+            ScanMailboxTask.onCancel[F]
+          )
+        )
         .withTask(
           JobTask.json(
             HouseKeepingTask.taskName,
diff --git a/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala b/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala
index e34134a1..cb1f3ce3 100644
--- a/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala
+++ b/modules/joex/src/main/scala/docspell/joex/notify/MailContext.scala
@@ -7,6 +7,7 @@ import docspell.common._
 import docspell.store.queries.QItem
 import docspell.joex.notify.YamuscaConverter._
 
+/** The context for rendering the e-mail template. */
 case class MailContext(
     items: List[MailContext.ItemData],
     more: Boolean,
diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
new file mode 100644
index 00000000..45dd1cf2
--- /dev/null
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -0,0 +1,55 @@
+package docspell.joex.scanmailbox
+
+import cats.implicits._
+import cats.effect._
+import emil._
+//import emil.javamail.syntax._
+
+import docspell.common._
+import docspell.store.records._
+import docspell.joex.scheduler.{Context, Task}
+
+object ScanMailboxTask {
+  val maxItems: Long = 7
+  type Args = ScanMailboxArgs
+
+  def apply[F[_]: Sync](emil: Emil[F]): Task[F, Args, Unit] =
+    Task { ctx =>
+      for {
+        _ <- ctx.logger.info(
+          s"Start importing mails for user ${ctx.args.account.user.id}"
+        )
+        mailCfg <- getMailSettings(ctx)
+        folders  = ctx.args.folders.mkString(", ")
+        userId   = ctx.args.account.user
+        imapConn = ctx.args.imapConnection
+        _ <- ctx.logger.info(
+          s"Reading mails for user ${userId.id} from ${imapConn.id}/${folders}"
+        )
+        _ <- importMails(mailCfg, emil, ctx)
+      } yield ()
+    }
+
+  def onCancel[F[_]: Sync]: Task[F, ScanMailboxArgs, Unit] =
+    Task.log(_.warn("Cancelling scan-mailbox task"))
+
+  def getMailSettings[F[_]: Sync](ctx: Context[F, Args]): F[RUserImap] =
+    ctx.store
+      .transact(RUserImap.getByName(ctx.args.account, ctx.args.imapConnection))
+      .flatMap {
+        case Some(c) => c.pure[F]
+        case None =>
+          Sync[F].raiseError(
+            new Exception(
+              s"No imap configuration found for: ${ctx.args.imapConnection.id}"
+            )
+          )
+      }
+
+  def importMails[F[_]: Sync](
+      cfg: RUserImap,
+      emil: Emil[F],
+      ctx: Context[F, Args]
+  ): F[Unit] =
+    Sync[F].delay(println(s"$emil $ctx $cfg"))
+}

From 0d6677f90bda2f2dbb8620908f622041351fdea1 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Mon, 18 May 2020 09:55:49 +0200
Subject: [PATCH 05/14] Add stub form scan-mailbox form

---
 .../src/main/resources/docspell-openapi.yml   | 105 +++++
 modules/webapp/src/main/elm/Api.elm           |  48 +++
 .../src/main/elm/Comp/ScanMailboxForm.elm     | 372 ++++++++++++++++++
 .../src/main/elm/Page/UserSettings/Data.elm   |   5 +
 .../src/main/elm/Page/UserSettings/Update.elm |  18 +
 .../src/main/elm/Page/UserSettings/View.elm   |  28 ++
 6 files changed, 576 insertions(+)
 create mode 100644 modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm

diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml
index 642a8bba..090a80d7 100644
--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -1755,8 +1755,113 @@ paths:
               schema:
                 $ref: "#/components/schemas/BasicResult"
 
+  /sec/usertask/scanmailbox:
+    get:
+      tags: [ User Tasks ]
+      summary: Get settings for "Scan Mailbox" task
+      description: |
+        Return the current settings for the scan mailbox task of the
+        authenticated user. Users can periodically fetch mails to be
+        imported into docspell.
+      security:
+        - authTokenHeader: []
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ScanMailboxSettings"
+    post:
+      tags: [ User Tasks ]
+      summary: Change current settings for "Scan Mailbox" task
+      description: |
+        Change the current settings for the scan-mailbox task of the
+        authenticated user.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ScanMailboxSettings"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
+  /sec/usertask/scanmailbox/startonce:
+    post:
+      tags: [ User Tasks ]
+      summary: Start the "Scan Mailbox" task once
+      description: |
+        Starts the scan-mailbox task just once, discarding the
+        schedule and not updating the periodic task.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ScanMailboxSettings"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
+
 components:
   schemas:
+    ScanMailboxSettings:
+      description: |
+        Settings for the scan mailbox task.
+      required:
+        - id
+        - enabled
+        - imapConnection
+        - schedule
+        - folders
+      properties:
+        id:
+          type: string
+          format: ident
+        enabled:
+          type: boolean
+        imapConnection:
+          type: string
+          format: ident
+        folders:
+          type: array
+          items:
+            type: string
+        schedule:
+          type: string
+          format: calevent
+        receivedSinceHours:
+          type: integer
+          description: |
+            Look only for mails newer than `receivedSinceHours' hours.
+        targetFolder:
+          type: string
+          description: |
+            The folder to move all mails into that have been
+            successfully submitted to docspell.
+        deleteMail:
+          type: boolean
+          description: |
+            Whether to delete all successfully imported mails. This
+            only applies, if `targetFolder' is not set.
+        direction:
+          type: string
+          format: direction
+          description: |
+            The direction to apply to items resulting from importing
+            mails. If not set, the value is guessed based on the from
+            and to mail headers and your address book.
     ImapSettingsList:
       description: |
         A list of user email settings.
diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm
index e1e74060..f56cb7d7 100644
--- a/modules/webapp/src/main/elm/Api.elm
+++ b/modules/webapp/src/main/elm/Api.elm
@@ -30,6 +30,7 @@ module Api exposing
     , getOrganizations
     , getPersons
     , getPersonsLight
+    , getScanMailbox
     , getSentMails
     , getSources
     , getTags
@@ -64,7 +65,9 @@ module Api exposing
     , setTags
     , setUnconfirmed
     , startOnceNotifyDueItems
+    , startOnceScanMailbox
     , submitNotifyDueItems
+    , submitScanMailbox
     , upload
     , uploadSingle
     , versionInfo
@@ -105,6 +108,7 @@ import Api.Model.Person exposing (Person)
 import Api.Model.PersonList exposing (PersonList)
 import Api.Model.ReferenceList exposing (ReferenceList)
 import Api.Model.Registration exposing (Registration)
+import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
 import Api.Model.SentMails exposing (SentMails)
 import Api.Model.SimpleMail exposing (SimpleMail)
 import Api.Model.Source exposing (Source)
@@ -127,6 +131,50 @@ import Util.Http as Http2
 
 
 
+--- Scan Mailboxes
+
+
+startOnceScanMailbox :
+    Flags
+    -> ScanMailboxSettings
+    -> (Result Http.Error BasicResult -> msg)
+    -> Cmd msg
+startOnceScanMailbox flags settings receive =
+    Http2.authPost
+        { url = flags.config.baseUrl ++ "/api/v1/sec/usertask/scanmailbox/startonce"
+        , account = getAccount flags
+        , body = Http.jsonBody (Api.Model.ScanMailboxSettings.encode settings)
+        , expect = Http.expectJson receive Api.Model.BasicResult.decoder
+        }
+
+
+submitScanMailbox :
+    Flags
+    -> ScanMailboxSettings
+    -> (Result Http.Error BasicResult -> msg)
+    -> Cmd msg
+submitScanMailbox flags settings receive =
+    Http2.authPost
+        { url = flags.config.baseUrl ++ "/api/v1/sec/usertask/scanmailbox"
+        , account = getAccount flags
+        , body = Http.jsonBody (Api.Model.ScanMailboxSettings.encode settings)
+        , expect = Http.expectJson receive Api.Model.BasicResult.decoder
+        }
+
+
+getScanMailbox :
+    Flags
+    -> (Result Http.Error ScanMailboxSettings -> msg)
+    -> Cmd msg
+getScanMailbox flags receive =
+    Http2.authGet
+        { url = flags.config.baseUrl ++ "/api/v1/sec/usertask/scanmailbox"
+        , account = getAccount flags
+        , expect = Http.expectJson receive Api.Model.ScanMailboxSettings.decoder
+        }
+
+
+
 --- NotifyDueItems
 
 
diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm
new file mode 100644
index 00000000..e1628039
--- /dev/null
+++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm
@@ -0,0 +1,372 @@
+module Comp.ScanMailboxForm exposing
+    ( Model
+    , Msg
+    , init
+    , update
+    , view
+    )
+
+import Api
+import Api.Model.BasicResult exposing (BasicResult)
+import Api.Model.ImapSettingsList exposing (ImapSettingsList)
+import Api.Model.ScanMailboxSettings exposing (ScanMailboxSettings)
+import Api.Model.Tag exposing (Tag)
+import Api.Model.TagList exposing (TagList)
+import Comp.CalEventInput
+import Comp.Dropdown
+import Comp.EmailInput
+import Comp.IntField
+import Data.CalEvent exposing (CalEvent)
+import Data.Flags exposing (Flags)
+import Data.Validated exposing (Validated(..))
+import Html exposing (..)
+import Html.Attributes exposing (..)
+import Html.Events exposing (onCheck, onClick)
+import Http
+import Util.Http
+import Util.Maybe
+import Util.Update
+
+
+type alias Model =
+    { settings : ScanMailboxSettings
+    , connectionModel : Comp.Dropdown.Model String
+    , enabled : Bool
+    , schedule : Validated CalEvent
+    , scheduleModel : Comp.CalEventInput.Model
+    , formMsg : Maybe BasicResult
+    , loading : Int
+    }
+
+
+type Msg
+    = Submit
+    | ConnMsg (Comp.Dropdown.Msg String)
+    | ConnResp (Result Http.Error ImapSettingsList)
+    | ToggleEnabled
+    | CalEventMsg Comp.CalEventInput.Msg
+    | SetScanMailboxSettings (Result Http.Error ScanMailboxSettings)
+    | SubmitResp (Result Http.Error BasicResult)
+    | StartOnce
+
+
+initCmd : Flags -> Cmd Msg
+initCmd flags =
+    Cmd.batch
+        [ Api.getImapSettings flags "" ConnResp
+        , Api.getScanMailbox flags SetScanMailboxSettings
+        ]
+
+
+init : Flags -> ( Model, Cmd Msg )
+init flags =
+    let
+        initialSchedule =
+            Data.Validated.Unknown Data.CalEvent.everyMonth
+
+        ( sm, sc ) =
+            Comp.CalEventInput.init flags Data.CalEvent.everyMonth
+    in
+    ( { settings = Api.Model.ScanMailboxSettings.empty
+      , connectionModel =
+            Comp.Dropdown.makeSingle
+                { makeOption = \a -> { value = a, text = a }
+                , placeholder = "Select connection..."
+                }
+      , enabled = False
+      , schedule = initialSchedule
+      , scheduleModel = sm
+      , formMsg = Nothing
+      , loading = 3
+      }
+    , Cmd.batch
+        [ initCmd flags
+        , Cmd.map CalEventMsg sc
+        ]
+    )
+
+
+
+--- Update
+
+
+makeSettings : Model -> Validated ScanMailboxSettings
+makeSettings model =
+    let
+        prev =
+            model.settings
+
+        conn =
+            Comp.Dropdown.getSelected model.connectionModel
+                |> List.head
+                |> Maybe.map Valid
+                |> Maybe.withDefault (Invalid [ "Connection missing" ] "")
+
+        make smtp timer =
+            { prev
+                | imapConnection = smtp
+                , enabled = model.enabled
+                , schedule = Data.CalEvent.makeEvent timer
+            }
+    in
+    Data.Validated.map2 make
+        conn
+        model.schedule
+
+
+withValidSettings : (ScanMailboxSettings -> Cmd Msg) -> Model -> ( Model, Cmd Msg )
+withValidSettings mkcmd model =
+    case makeSettings model of
+        Valid set ->
+            ( { model | formMsg = Nothing }
+            , mkcmd set
+            )
+
+        Invalid errs _ ->
+            let
+                errMsg =
+                    String.join ", " errs
+            in
+            ( { model | formMsg = Just (BasicResult False errMsg) }, Cmd.none )
+
+        Unknown _ ->
+            ( { model | formMsg = Just (BasicResult False "An unknown error occured") }
+            , Cmd.none
+            )
+
+
+update : Flags -> Msg -> Model -> ( Model, Cmd Msg )
+update flags msg model =
+    case msg of
+        CalEventMsg lmsg ->
+            let
+                ( cm, cc, cs ) =
+                    Comp.CalEventInput.update flags
+                        (Data.Validated.value model.schedule)
+                        lmsg
+                        model.scheduleModel
+            in
+            ( { model
+                | schedule = cs
+                , scheduleModel = cm
+                , formMsg = Nothing
+              }
+            , Cmd.map CalEventMsg cc
+            )
+
+        ConnMsg m ->
+            let
+                ( cm, cc ) =
+                    Comp.Dropdown.update m model.connectionModel
+            in
+            ( { model
+                | connectionModel = cm
+                , formMsg = Nothing
+              }
+            , Cmd.map ConnMsg cc
+            )
+
+        ConnResp (Ok list) ->
+            let
+                names =
+                    List.map .name list.items
+
+                cm =
+                    Comp.Dropdown.makeSingleList
+                        { makeOption = \a -> { value = a, text = a }
+                        , placeholder = "Select Connection..."
+                        , options = names
+                        , selected = List.head names
+                        }
+            in
+            ( { model
+                | connectionModel = cm
+                , loading = model.loading - 1
+                , formMsg =
+                    if names == [] then
+                        Just
+                            (BasicResult False
+                                "No E-Mail connections configured. Goto E-Mail Settings to add one."
+                            )
+
+                    else
+                        Nothing
+              }
+            , Cmd.none
+            )
+
+        ConnResp (Err err) ->
+            ( { model
+                | formMsg = Just (BasicResult False (Util.Http.errorToString err))
+                , loading = model.loading - 1
+              }
+            , Cmd.none
+            )
+
+        ToggleEnabled ->
+            ( { model
+                | enabled = not model.enabled
+                , formMsg = Nothing
+              }
+            , Cmd.none
+            )
+
+        SetScanMailboxSettings (Ok s) ->
+            let
+                imap =
+                    Util.Maybe.fromString s.imapConnection
+                        |> Maybe.map List.singleton
+                        |> Maybe.withDefault []
+
+                ( nm, nc ) =
+                    Util.Update.andThen1
+                        [ update flags (ConnMsg (Comp.Dropdown.SetSelection imap))
+                        ]
+                        model
+
+                newSchedule =
+                    Data.CalEvent.fromEvent s.schedule
+                        |> Maybe.withDefault Data.CalEvent.everyMonth
+
+                ( sm, sc ) =
+                    Comp.CalEventInput.init flags newSchedule
+            in
+            ( { nm
+                | settings = s
+                , enabled = s.enabled
+                , schedule = Data.Validated.Unknown newSchedule
+                , scheduleModel = sm
+                , formMsg = Nothing
+                , loading = model.loading - 1
+              }
+            , Cmd.batch
+                [ nc
+                , Cmd.map CalEventMsg sc
+                ]
+            )
+
+        SetScanMailboxSettings (Err err) ->
+            ( { model
+                | formMsg = Just (BasicResult False (Util.Http.errorToString err))
+                , loading = model.loading - 1
+              }
+            , Cmd.none
+            )
+
+        Submit ->
+            withValidSettings
+                (\set -> Api.submitScanMailbox flags set SubmitResp)
+                model
+
+        StartOnce ->
+            withValidSettings
+                (\set -> Api.startOnceScanMailbox flags set SubmitResp)
+                model
+
+        SubmitResp (Ok res) ->
+            ( { model | formMsg = Just res }
+            , Cmd.none
+            )
+
+        SubmitResp (Err err) ->
+            ( { model
+                | formMsg = Just (BasicResult False (Util.Http.errorToString err))
+              }
+            , Cmd.none
+            )
+
+
+
+--- View
+
+
+isFormError : Model -> Bool
+isFormError model =
+    Maybe.map .success model.formMsg
+        |> Maybe.map not
+        |> Maybe.withDefault False
+
+
+isFormSuccess : Model -> Bool
+isFormSuccess model =
+    Maybe.map .success model.formMsg
+        |> Maybe.withDefault False
+
+
+view : String -> Model -> Html Msg
+view extraClasses model =
+    div
+        [ classList
+            [ ( "ui form", True )
+            , ( extraClasses, True )
+            , ( "error", isFormError model )
+            , ( "success", isFormSuccess model )
+            ]
+        ]
+        [ div
+            [ classList
+                [ ( "ui dimmer", True )
+                , ( "active", model.loading > 0 )
+                ]
+            ]
+            [ div [ class "ui text loader" ]
+                [ text "Loading..."
+                ]
+            ]
+        , div [ class "required field" ]
+            [ label [] [ text "Send via" ]
+            , Html.map ConnMsg (Comp.Dropdown.view model.connectionModel)
+            , span [ class "small-info" ]
+                [ text "The IMAP connection to use when sending notification mails."
+                ]
+            ]
+        , div [ class "required field" ]
+            [ label []
+                [ text "Schedule"
+                , a
+                    [ class "right-float"
+                    , href "https://github.com/eikek/calev#what-are-calendar-events"
+                    , target "_blank"
+                    ]
+                    [ i [ class "help icon" ] []
+                    , text "Click here for help"
+                    ]
+                ]
+            , Html.map CalEventMsg
+                (Comp.CalEventInput.view ""
+                    (Data.Validated.value model.schedule)
+                    model.scheduleModel
+                )
+            , span [ class "small-info" ]
+                [ text "Specify how often and when this task should run. "
+                , text "Use English 3-letter weekdays. Either a single value, "
+                , text "a list (ex. 1,2,3), a range (ex. 1..3) or a '*' (meaning all) "
+                , text "is allowed for each part."
+                ]
+            ]
+        , div [ class "ui divider" ] []
+        , div
+            [ classList
+                [ ( "ui message", True )
+                , ( "success", isFormSuccess model )
+                , ( "error", isFormError model )
+                , ( "hidden", model.formMsg == Nothing )
+                ]
+            ]
+            [ Maybe.map .message model.formMsg
+                |> Maybe.withDefault ""
+                |> text
+            ]
+        , button
+            [ class "ui primary button"
+            , onClick Submit
+            ]
+            [ text "Submit"
+            ]
+        , button
+            [ class "ui right floated button"
+            , onClick StartOnce
+            ]
+            [ text "Start Once"
+            ]
+        ]
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Data.elm b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
index 56276a09..292fc5b0 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm
@@ -9,6 +9,7 @@ import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
 import Comp.ImapSettingsManage
 import Comp.NotificationForm
+import Comp.ScanMailboxForm
 import Data.Flags exposing (Flags)
 
 
@@ -18,6 +19,7 @@ type alias Model =
     , emailSettingsModel : Comp.EmailSettingsManage.Model
     , imapSettingsModel : Comp.ImapSettingsManage.Model
     , notificationModel : Comp.NotificationForm.Model
+    , scanMailboxModel : Comp.ScanMailboxForm.Model
     }
 
 
@@ -28,6 +30,7 @@ emptyModel flags =
     , emailSettingsModel = Comp.EmailSettingsManage.emptyModel
     , imapSettingsModel = Comp.ImapSettingsManage.emptyModel
     , notificationModel = Tuple.first (Comp.NotificationForm.init flags)
+    , scanMailboxModel = Tuple.first (Comp.ScanMailboxForm.init flags)
     }
 
 
@@ -36,6 +39,7 @@ type Tab
     | EmailSettingsTab
     | ImapSettingsTab
     | NotificationTab
+    | ScanMailboxTab
 
 
 type Msg
@@ -44,3 +48,4 @@ type Msg
     | EmailSettingsMsg Comp.EmailSettingsManage.Msg
     | NotificationMsg Comp.NotificationForm.Msg
     | ImapSettingsMsg Comp.ImapSettingsManage.Msg
+    | ScanMailboxMsg Comp.ScanMailboxForm.Msg
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Update.elm b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
index 1a56cac1..fffa6b8c 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm
@@ -4,6 +4,7 @@ import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
 import Comp.ImapSettingsManage
 import Comp.NotificationForm
+import Comp.ScanMailboxForm
 import Data.Flags exposing (Flags)
 import Page.UserSettings.Data exposing (..)
 
@@ -42,6 +43,14 @@ update flags msg model =
                                         (Tuple.second (Comp.NotificationForm.init flags))
                             in
                             ( m, initCmd )
+
+                        ScanMailboxTab ->
+                            let
+                                initCmd =
+                                    Cmd.map ScanMailboxMsg
+                                        (Tuple.second (Comp.ScanMailboxForm.init flags))
+                            in
+                            ( m, initCmd )
             in
             ( m2, cmd )
 
@@ -74,3 +83,12 @@ update flags msg model =
             ( { model | notificationModel = m2 }
             , Cmd.map NotificationMsg c2
             )
+
+        ScanMailboxMsg lm ->
+            let
+                ( m2, c2 ) =
+                    Comp.ScanMailboxForm.update flags lm model.scanMailboxModel
+            in
+            ( { model | scanMailboxModel = m2 }
+            , Cmd.map ScanMailboxMsg c2
+            )
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
index 4997c5ee..2fad7e0b 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/View.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
@@ -4,6 +4,7 @@ import Comp.ChangePasswordForm
 import Comp.EmailSettingsManage
 import Comp.ImapSettingsManage
 import Comp.NotificationForm
+import Comp.ScanMailboxForm
 import Html exposing (..)
 import Html.Attributes exposing (..)
 import Html.Events exposing (onClick)
@@ -24,6 +25,7 @@ view model =
                     , makeTab model EmailSettingsTab "E-Mail Settings (SMTP)" "mail icon"
                     , makeTab model ImapSettingsTab "E-Mail Settings (IMAP)" "mail icon"
                     , makeTab model NotificationTab "Notification Task" "bullhorn icon"
+                    , makeTab model ScanMailboxTab "Scan Mailbox Task" "envelope open outline icon"
                     ]
                 ]
             ]
@@ -42,6 +44,9 @@ view model =
                     Just ImapSettingsTab ->
                         viewImapSettings model
 
+                    Just ScanMailboxTab ->
+                        viewScanMailboxForm model
+
                     Nothing ->
                         []
                 )
@@ -118,3 +123,26 @@ viewNotificationForm model =
     , Html.map NotificationMsg
         (Comp.NotificationForm.view "segment" model.notificationModel)
     ]
+
+
+viewScanMailboxForm : Model -> List (Html Msg)
+viewScanMailboxForm model =
+    [ h2 [ class "ui header" ]
+        [ i [ class "ui bullhorn icon" ] []
+        , div [ class "content" ]
+            [ text "Scan Mailbox"
+            ]
+        ]
+    , p []
+        [ text "Docspell can scan folders of your mailbox for mails to import. "
+        , text "You need to provide a connection in "
+        , text "your e-mail (imap) settings."
+        ]
+    , p []
+        [ text "Each time this is executed, docspell goes through all configured folders "
+        , text "and imports mails matching the search criteria. The number of mails to import "
+        , text "at one task run is limited. Mails already read in are skipped."
+        ]
+    , Html.map ScanMailboxMsg
+        (Comp.ScanMailboxForm.view "segment" model.scanMailboxModel)
+    ]

From 6e8582ea8079affec7ce65cd269e13e126c2e044 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Mon, 18 May 2020 11:43:18 +0200
Subject: [PATCH 06/14] Implement scan-mailbox form and routes

---
 .../docspell/backend/ops/OUserTask.scala      |  47 ++++
 .../main/scala/docspell/common/Duration.scala |   5 +-
 .../src/main/resources/docspell-openapi.yml   |   2 +
 .../docspell/restserver/RestServer.scala      |   1 +
 .../restserver/routes/ScanMailboxRoutes.scala | 103 +++++++++
 .../src/main/elm/Comp/ScanMailboxForm.elm     | 202 +++++++++++++++++-
 .../src/main/elm/Comp/StringListInput.elm     |  98 +++++++++
 .../src/main/elm/Page/UserSettings/View.elm   |   2 +-
 8 files changed, 452 insertions(+), 8 deletions(-)
 create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala
 create mode 100644 modules/webapp/src/main/elm/Comp/StringListInput.elm

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala
index d002d654..26233eb1 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala
@@ -11,6 +11,18 @@ import docspell.common._
 
 trait OUserTask[F[_]] {
 
+  /** Return the settings for the scan-mailbox task of the current user.
+    * There is at most one such task per user.
+    */
+  def getScanMailbox(account: AccountId): F[UserTask[ScanMailboxArgs]]
+
+  /** Updates the scan-mailbox tasks and notifies the joex nodes.
+    */
+  def submitScanMailbox(
+      account: AccountId,
+      task: UserTask[ScanMailboxArgs]
+  ): F[Unit]
+
   /** Return the settings for the notify-due-items task of the current
     * user. There is at most one such task per user.
     */
@@ -51,6 +63,20 @@ object OUserTask {
           _     <- joex.notifyAllNodes
         } yield ()
 
+      def getScanMailbox(account: AccountId): F[UserTask[ScanMailboxArgs]] =
+        store
+          .getOneByName[ScanMailboxArgs](account, ScanMailboxArgs.taskName)
+          .getOrElseF(scanMailboxDefault(account))
+
+      def submitScanMailbox(
+          account: AccountId,
+          task: UserTask[ScanMailboxArgs]
+      ): F[Unit] =
+        for {
+          _ <- store.updateOneTask[ScanMailboxArgs](account, task)
+          _ <- joex.notifyAllNodes
+        } yield ()
+
       def getNotifyDueItems(account: AccountId): F[UserTask[NotifyDueItemsArgs]] =
         store
           .getOneByName[NotifyDueItemsArgs](account, NotifyDueItemsArgs.taskName)
@@ -86,6 +112,27 @@ object OUserTask {
             Nil
           )
         )
+
+      private def scanMailboxDefault(
+          account: AccountId
+      ): F[UserTask[ScanMailboxArgs]] =
+        for {
+          id <- Ident.randomId[F]
+        } yield UserTask(
+          id,
+          ScanMailboxArgs.taskName,
+          false,
+          CalEvent.unsafe("*-*-* 0,12:00"),
+          ScanMailboxArgs(
+            account,
+            Ident.unsafe(""),
+            Nil,
+            Some(Duration.hours(12)),
+            None,
+            false,
+            None
+          )
+        )
     })
 
 }
diff --git a/modules/common/src/main/scala/docspell/common/Duration.scala b/modules/common/src/main/scala/docspell/common/Duration.scala
index 1f837196..bb47059e 100644
--- a/modules/common/src/main/scala/docspell/common/Duration.scala
+++ b/modules/common/src/main/scala/docspell/common/Duration.scala
@@ -13,6 +13,10 @@ case class Duration(nanos: Long) {
 
   def seconds: Long = millis / 1000
 
+  def minutes: Long = seconds / 60
+
+  def hours: Long = minutes / 60
+
   def toScala: FiniteDuration =
     FiniteDuration(nanos, TimeUnit.NANOSECONDS)
 
@@ -55,7 +59,6 @@ object Duration {
       end = Timestamp.current[F]
     } yield end.map(e => Duration.millis(e.toMillis - now.toMillis))
 
-
   implicit val jsonEncoder: Encoder[Duration] =
     Encoder.encodeLong.contramap(_.millis)
 
diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml
index 090a80d7..6c4a4103 100644
--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -1825,6 +1825,7 @@ components:
         - imapConnection
         - schedule
         - folders
+        - deleteMail
       properties:
         id:
           type: string
@@ -1843,6 +1844,7 @@ components:
           format: calevent
         receivedSinceHours:
           type: integer
+          format: int32
           description: |
             Look only for mails newer than `receivedSinceHours' hours.
         targetFolder:
diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
index 1a9266de..02008552 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala
@@ -76,6 +76,7 @@ object RestServer {
       "email/settings"          -> MailSettingsRoutes(restApp.backend, token),
       "email/sent"              -> SentMailRoutes(restApp.backend, token),
       "usertask/notifydueitems" -> NotifyDueItemsRoutes(cfg, restApp.backend, token),
+      "usertask/scanmailbox"    -> ScanMailboxRoutes(restApp.backend, token),
       "calevent/check"          -> CalEventCheckRoutes()
     )
 
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala
new file mode 100644
index 00000000..5c7d0fa5
--- /dev/null
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala
@@ -0,0 +1,103 @@
+package docspell.restserver.routes
+
+import cats.effect._
+import cats.implicits._
+import org.http4s._
+import org.http4s.dsl.Http4sDsl
+import org.http4s.circe.CirceEntityEncoder._
+import org.http4s.circe.CirceEntityDecoder._
+
+import docspell.backend.BackendApp
+import docspell.backend.auth.AuthToken
+import docspell.common._
+import docspell.restapi.model._
+import docspell.store.usertask._
+import docspell.restserver.conv.Conversions
+
+object ScanMailboxRoutes {
+
+  def apply[F[_]: Effect](
+      backend: BackendApp[F],
+      user: AuthToken
+  ): HttpRoutes[F] = {
+    val dsl = new Http4sDsl[F] {}
+    val ut  = backend.userTask
+    import dsl._
+
+    HttpRoutes.of {
+      case req @ POST -> Root / "startonce" =>
+        for {
+          data <- req.as[ScanMailboxSettings]
+          task = makeTask(user.account, data)
+          res <-
+            ut.executeNow(user.account, task)
+              .attempt
+              .map(Conversions.basicResult(_, "Submitted successfully."))
+          resp <- Ok(res)
+        } yield resp
+
+      case GET -> Root =>
+        for {
+          task <- ut.getScanMailbox(user.account)
+          res  <- taskToSettings(user.account, backend, task)
+          resp <- Ok(res)
+        } yield resp
+
+      case req @ POST -> Root =>
+        for {
+          data <- req.as[ScanMailboxSettings]
+          task = makeTask(user.account, data)
+          res <-
+            ut.submitScanMailbox(user.account, task)
+              .attempt
+              .map(Conversions.basicResult(_, "Saved successfully."))
+          resp <- Ok(res)
+        } yield resp
+    }
+  }
+
+  def makeTask(
+      user: AccountId,
+      settings: ScanMailboxSettings
+  ): UserTask[ScanMailboxArgs] =
+    UserTask(
+      settings.id,
+      ScanMailboxArgs.taskName,
+      settings.enabled,
+      settings.schedule,
+      ScanMailboxArgs(
+        user,
+        settings.imapConnection,
+        settings.folders,
+        settings.receivedSinceHours.map(_.toLong).map(Duration.hours),
+        settings.targetFolder,
+        settings.deleteMail,
+        settings.direction
+      )
+    )
+
+  def taskToSettings[F[_]: Sync](
+      account: AccountId,
+      backend: BackendApp[F],
+      task: UserTask[ScanMailboxArgs]
+  ): F[ScanMailboxSettings] =
+    for {
+      conn <-
+        backend.mail
+          .getImapSettings(account, None)
+          .map(
+            _.find(_.name == task.args.imapConnection)
+              .map(_.name)
+          )
+    } yield ScanMailboxSettings(
+      task.id,
+      task.enabled,
+      conn.getOrElse(Ident.unsafe("")),
+      task.args.folders, //folders
+      task.timer,
+      task.args.receivedSince.map(_.hours.toInt),
+      task.args.targetFolder,
+      task.args.deleteMail,
+      task.args.direction
+    )
+}
diff --git a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm
index e1628039..e44d9845 100644
--- a/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm
+++ b/modules/webapp/src/main/elm/Comp/ScanMailboxForm.elm
@@ -14,16 +14,18 @@ import Api.Model.Tag exposing (Tag)
 import Api.Model.TagList exposing (TagList)
 import Comp.CalEventInput
 import Comp.Dropdown
-import Comp.EmailInput
 import Comp.IntField
+import Comp.StringListInput
 import Data.CalEvent exposing (CalEvent)
+import Data.Direction exposing (Direction(..))
 import Data.Flags exposing (Flags)
 import Data.Validated exposing (Validated(..))
 import Html exposing (..)
 import Html.Attributes exposing (..)
-import Html.Events exposing (onCheck, onClick)
+import Html.Events exposing (onCheck, onClick, onInput)
 import Http
 import Util.Http
+import Util.List
 import Util.Maybe
 import Util.Update
 
@@ -32,6 +34,13 @@ type alias Model =
     { settings : ScanMailboxSettings
     , connectionModel : Comp.Dropdown.Model String
     , enabled : Bool
+    , deleteMail : Bool
+    , receivedHours : Maybe Int
+    , receivedHoursModel : Comp.IntField.Model
+    , targetFolder : Maybe String
+    , foldersModel : Comp.StringListInput.Model
+    , folders : List String
+    , direction : Maybe Direction
     , schedule : Validated CalEvent
     , scheduleModel : Comp.CalEventInput.Model
     , formMsg : Maybe BasicResult
@@ -44,10 +53,15 @@ type Msg
     | ConnMsg (Comp.Dropdown.Msg String)
     | ConnResp (Result Http.Error ImapSettingsList)
     | ToggleEnabled
+    | ToggleDeleteMail
     | CalEventMsg Comp.CalEventInput.Msg
     | SetScanMailboxSettings (Result Http.Error ScanMailboxSettings)
     | SubmitResp (Result Http.Error BasicResult)
     | StartOnce
+    | ReceivedHoursMsg Comp.IntField.Msg
+    | SetTargetFolder String
+    | FoldersMsg Comp.StringListInput.Msg
+    | DirectionMsg (Maybe Direction)
 
 
 initCmd : Flags -> Cmd Msg
@@ -74,10 +88,17 @@ init flags =
                 , placeholder = "Select connection..."
                 }
       , enabled = False
+      , deleteMail = False
+      , receivedHours = Nothing
+      , receivedHoursModel = Comp.IntField.init (Just 1) Nothing True "Received Since Hours"
+      , foldersModel = Comp.StringListInput.init
+      , folders = []
+      , targetFolder = Nothing
+      , direction = Nothing
       , schedule = initialSchedule
       , scheduleModel = sm
       , formMsg = Nothing
-      , loading = 3
+      , loading = 2
       }
     , Cmd.batch
         [ initCmd flags
@@ -102,16 +123,29 @@ makeSettings model =
                 |> Maybe.map Valid
                 |> Maybe.withDefault (Invalid [ "Connection missing" ] "")
 
-        make smtp timer =
+        infolders =
+            if model.folders == [] then
+                Invalid [ "No folders given" ] []
+
+            else
+                Valid model.folders
+
+        make smtp timer folders =
             { prev
                 | imapConnection = smtp
                 , enabled = model.enabled
+                , receivedSinceHours = model.receivedHours
+                , deleteMail = model.deleteMail
+                , targetFolder = model.targetFolder
+                , folders = folders
+                , direction = Maybe.map Data.Direction.toString model.direction
                 , schedule = Data.CalEvent.makeEvent timer
             }
     in
-    Data.Validated.map2 make
+    Data.Validated.map3 make
         conn
         model.schedule
+        infolders
 
 
 withValidSettings : (ScanMailboxSettings -> Cmd Msg) -> Model -> ( Model, Cmd Msg )
@@ -211,6 +245,60 @@ update flags msg model =
             , Cmd.none
             )
 
+        ToggleDeleteMail ->
+            ( { model
+                | deleteMail = not model.deleteMail
+                , formMsg = Nothing
+              }
+            , Cmd.none
+            )
+
+        ReceivedHoursMsg m ->
+            let
+                ( pm, val ) =
+                    Comp.IntField.update m model.receivedHoursModel
+            in
+            ( { model
+                | receivedHoursModel = pm
+                , receivedHours = val
+                , formMsg = Nothing
+              }
+            , Cmd.none
+            )
+
+        SetTargetFolder str ->
+            ( { model | targetFolder = Util.Maybe.fromString str }
+            , Cmd.none
+            )
+
+        FoldersMsg lm ->
+            let
+                ( fm, itemAction ) =
+                    Comp.StringListInput.update lm model.foldersModel
+
+                newList =
+                    case itemAction of
+                        Comp.StringListInput.AddAction s ->
+                            Util.List.distinct (s :: model.folders)
+
+                        Comp.StringListInput.RemoveAction s ->
+                            List.filter (\e -> e /= s) model.folders
+
+                        Comp.StringListInput.NoAction ->
+                            model.folders
+            in
+            ( { model
+                | foldersModel = fm
+                , folders = newList
+              }
+            , Cmd.none
+            )
+
+        DirectionMsg md ->
+            ( { model | direction = md }
+            , Cmd.none
+            )
+
         SetScanMailboxSettings (Ok s) ->
             let
                 imap =
@@ -234,7 +322,12 @@ update flags msg model =
             ( { nm
                 | settings = s
                 , enabled = s.enabled
+                , deleteMail = s.deleteMail
+                , receivedHours = s.receivedSinceHours
+                , targetFolder = s.targetFolder
+                , folders = s.folders
                 , schedule = Data.Validated.Unknown newSchedule
+                , direction = Maybe.andThen Data.Direction.fromString s.direction
                 , scheduleModel = sm
                 , formMsg = Nothing
                 , loading = model.loading - 1
@@ -313,13 +406,110 @@ view extraClasses model =
                 [ text "Loading..."
                 ]
             ]
+        , div [ class "inline field" ]
+            [ div [ class "ui checkbox" ]
+                [ input
+                    [ type_ "checkbox"
+                    , onCheck (\_ -> ToggleEnabled)
+                    , checked model.enabled
+                    ]
+                    []
+                , label [] [ text "Enabled" ]
+                ]
+            , span [ class "small-info" ]
+                [ text "Enable or disable this task."
+                ]
+            ]
         , div [ class "required field" ]
-            [ label [] [ text "Send via" ]
+            [ label [] [ text "Mailbox" ]
             , Html.map ConnMsg (Comp.Dropdown.view model.connectionModel)
             , span [ class "small-info" ]
                 [ text "The IMAP connection to use when sending notification mails."
                 ]
             ]
+        , div [ class "required field" ]
+            [ label [] [ text "Folders" ]
+            , Html.map FoldersMsg (Comp.StringListInput.view model.folders model.foldersModel)
+            , span [ class "small-info" ]
+                [ text "The folders to go through"
+                ]
+            ]
+        , Html.map ReceivedHoursMsg
+            (Comp.IntField.viewWithInfo
+                "Select mails newer than `now - receivedHours`"
+                model.receivedHours
+                "field"
+                model.receivedHoursModel
+            )
+        , div [ class "field" ]
+            [ label [] [ text "Target folder" ]
+            , input
+                [ type_ "text"
+                , onInput SetTargetFolder
+                , Maybe.withDefault "" model.targetFolder |> value
+                ]
+                []
+            , span [ class "small-info" ]
+                [ text "Move all mails successfully submitted into this folder."
+                ]
+            ]
+        , div [ class "inline field" ]
+            [ div [ class "ui checkbox" ]
+                [ input
+                    [ type_ "checkbox"
+                    , onCheck (\_ -> ToggleDeleteMail)
+                    , checked model.deleteMail
+                    ]
+                    []
+                , label [] [ text "Delete imported mails" ]
+                ]
+            , span [ class "small-info" ]
+                [ text "Whether to delete all mails successfully imported into docspell."
+                ]
+            ]
+        , div [ class "required field" ]
+            [ label [] [ text "Item direction" ]
+            , div [ class "grouped fields" ]
+                [ div [ class "field" ]
+                    [ div [ class "ui radio checkbox" ]
+                        [ input
+                            [ type_ "radio"
+                            , checked (model.direction == Nothing)
+                            , onCheck (\_ -> DirectionMsg Nothing)
+                            ]
+                            []
+                        , label [] [ text "Automatic" ]
+                        ]
+                    ]
+                , div [ class "field" ]
+                    [ div [ class "ui radio checkbox" ]
+                        [ input
+                            [ type_ "radio"
+                            , checked (model.direction == Just Incoming)
+                            , onCheck (\_ -> DirectionMsg (Just Incoming))
+                            ]
+                            []
+                        , label [] [ text "Incoming" ]
+                        ]
+                    ]
+                , div [ class "field" ]
+                    [ div [ class "ui radio checkbox" ]
+                        [ input
+                            [ type_ "radio"
+                            , checked (model.direction == Just Outgoing)
+                            , onCheck (\_ -> DirectionMsg (Just Outgoing))
+                            ]
+                            []
+                        , label [] [ text "Outgoing" ]
+                        ]
+                    ]
+                , span [ class "small-info" ]
+                    [ text "Sets the direction for an item. If you know all mails are incoming or "
+                    , text "outgoing, you can set it here. Otherwise it will be guessed from looking "
+                    , text "at sender and receiver."
+                    ]
+                ]
+            ]
         , div [ class "required field" ]
             [ label []
                 [ text "Schedule"
diff --git a/modules/webapp/src/main/elm/Comp/StringListInput.elm b/modules/webapp/src/main/elm/Comp/StringListInput.elm
new file mode 100644
index 00000000..9c5b77c1
--- /dev/null
+++ b/modules/webapp/src/main/elm/Comp/StringListInput.elm
@@ -0,0 +1,98 @@
+module Comp.StringListInput exposing
+    ( ItemAction(..)
+    , Model
+    , Msg
+    , init
+    , update
+    , view
+    )
+
+import Html exposing (..)
+import Html.Attributes exposing (..)
+import Html.Events exposing (onClick, onInput)
+import Util.Maybe
+
+
+type alias Model =
+    { currentInput : String
+    }
+
+
+type Msg
+    = AddString
+    | RemoveString String
+    | SetString String
+
+
+init : Model
+init =
+    { currentInput = ""
+    }
+
+
+
+--- Update
+
+
+type ItemAction
+    = AddAction String
+    | RemoveAction String
+    | NoAction
+
+
+update : Msg -> Model -> ( Model, ItemAction )
+update msg model =
+    case msg of
+        SetString str ->
+            ( { model | currentInput = str }
+            , NoAction
+            )
+
+        AddString ->
+            ( { model | currentInput = "" }
+            , Util.Maybe.fromString model.currentInput
+                |> Maybe.map AddAction
+                |> Maybe.withDefault NoAction
+            )
+
+        RemoveString s ->
+            ( model, RemoveAction s )
+
+
+
+--- View
+
+
+view : List String -> Model -> Html Msg
+view values model =
+    let
+        valueItem s =
+            div [ class "item" ]
+                [ a
+                    [ class "ui icon link"
+                    , onClick (RemoveString s)
+                    , href "#"
+                    ]
+                    [ i [ class "delete icon" ] []
+                    ]
+                , text s
+                ]
+    in
+    div [ class "string-list-input" ]
+        [ div [ class "ui list" ]
+            (List.map valueItem values)
+        , div [ class "ui icon input" ]
+            [ input
+                [ placeholder ""
+                , type_ "text"
+                , onInput SetString
+                , value model.currentInput
+                ]
+                []
+            , i
+                [ class "circular add link icon"
+                , onClick AddString
+                ]
+                []
+            ]
+        ]
diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
index 2fad7e0b..0ea0a609 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/View.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
@@ -128,7 +128,7 @@ viewNotificationForm model =
 viewScanMailboxForm : Model -> List (Html Msg)
 viewScanMailboxForm model =
     [ h2 [ class "ui header" ]
-        [ i [ class "ui bullhorn icon" ] []
+        [ i [ class "ui envelope open outline icon" ] []
         , div [ class "content" ]
             [ text "Scan Mailbox"
             ]

From 852455c610c8b3817da9f8ef2ae19a0c7b374b0b Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Mon, 18 May 2020 15:58:31 +0200
Subject: [PATCH 07/14] Add upload operation to task arguments

---
 .../scala/docspell/backend/BackendApp.scala   |  2 +-
 .../scala/docspell/backend/ops/OUpload.scala  |  4 +--
 .../joex/src/main/resources/reference.conf    | 24 +++++++++++++++++
 .../src/main/scala/docspell/joex/Config.scala |  4 ++-
 .../scala/docspell/joex/JoexAppImpl.scala     |  6 +++--
 .../joex/scanmailbox/ScanMailboxTask.scala    | 26 ++++++++++++++++---
 6 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala
index 331d79d4..f86bf39e 100644
--- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala
+++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala
@@ -48,7 +48,7 @@ object BackendApp {
       equipImpl    <- OEquipment[F](store)
       orgImpl      <- OOrganization(store)
       joexImpl     <- OJoex.create(httpClientEc, store)
-      uploadImpl   <- OUpload(store, queue, cfg, joexImpl)
+      uploadImpl   <- OUpload(store, queue, cfg.files, joexImpl)
       nodeImpl     <- ONode(store)
       jobImpl      <- OJob(store, joexImpl)
       itemImpl     <- OItem(store)
diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
index a99fd305..3130e635 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -52,7 +52,7 @@ object OUpload {
   def apply[F[_]: Sync](
       store: Store[F],
       queue: JobQueue[F],
-      cfg: Config,
+      cfg: Config.Files,
       joex: OJoex[F]
   ): Resource[F, OUpload[F]] =
     Resource.pure[F, OUpload[F]](new OUpload[F] {
@@ -105,7 +105,7 @@ object OUpload {
       private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
         logger.finfo(s"Receiving file $file") *>
           store.bitpeace
-            .saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None)
+            .saveNew(file.data, cfg.chunkSize, MimetypeHint(file.name, None), None)
             .compile
             .lastOrError
             .map(fm => Ident.unsafe(fm.id))
diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf
index ff90f24f..fc9f7892 100644
--- a/modules/joex/src/main/resources/reference.conf
+++ b/modules/joex/src/main/resources/reference.conf
@@ -308,4 +308,28 @@ docspell.joex {
       working-dir = ${java.io.tmpdir}"/docspell-convert"
     }
   }
+
+  # The same section is also present in the rest-server config. It is
+  # used when submitting files into the job queue for processing.
+  #
+  # Currently, these settings may affect memory usage of all nodes, so
+  # it should be the same on all nodes.
+  files {
+    # Defines the chunk size (in bytes) used to store the files.
+    # This will affect the memory footprint when uploading and
+    # downloading files. At most this amount is loaded into RAM for
+    # down- and uploading.
+    #
+    # It also defines the chunk size used for the blobs inside the
+    # database.
+    chunk-size = 524288
+
+    # The file content types that are considered valid. Docspell
+    # will only pass these files to processing. The processing code
+    # itself has also checks for which files are supported and which
+    # not. This affects the uploading part and can be used to
+    # restrict file types that should be handed over to processing.
+    # By default all files are allowed.
+    valid-mime-types = [ ]
+  }
 }
\ No newline at end of file
diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala
index 98280392..58a9cb31 100644
--- a/modules/joex/src/main/scala/docspell/joex/Config.scala
+++ b/modules/joex/src/main/scala/docspell/joex/Config.scala
@@ -7,6 +7,7 @@ import docspell.store.JdbcConfig
 import docspell.convert.ConvertConfig
 import docspell.extract.ExtractConfig
 import docspell.joex.hk.HouseKeepingConfig
+import docspell.backend.Config.Files
 
 case class Config(
     appId: Ident,
@@ -19,7 +20,8 @@ case class Config(
     extraction: ExtractConfig,
     textAnalysis: TextAnalysisConfig,
     convert: ConvertConfig,
-    sendMail: MailSendConfig
+    sendMail: MailSendConfig,
+    files: Files
 )
 
 object Config {
diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
index ac6ff7c7..354c81cd 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
@@ -4,6 +4,7 @@ import cats.implicits._
 import cats.effect._
 import emil.javamail._
 import docspell.common._
+import docspell.backend.ops._
 import docspell.joex.hk._
 import docspell.joex.notify._
 import docspell.joex.scanmailbox._
@@ -12,7 +13,6 @@ import docspell.joex.scheduler._
 import docspell.joexapi.client.JoexClient
 import docspell.store.Store
 import docspell.store.queue._
-import docspell.backend.ops.ONode
 import docspell.store.records.RJobLog
 import fs2.concurrent.SignallingRef
 import scala.concurrent.ExecutionContext
@@ -68,6 +68,8 @@ object JoexAppImpl {
       queue   <- JobQueue(store)
       pstore  <- PeriodicTaskStore.create(store)
       nodeOps <- ONode(store)
+      joex    <- OJoex(client, store)
+      upload  <- OUpload(store, queue, cfg.files, joex)
       javaEmil = JavaMailEmil(blocker)
       sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
         .withQueue(queue)
@@ -88,7 +90,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             ScanMailboxArgs.taskName,
-            ScanMailboxTask[F](javaEmil),
+            ScanMailboxTask[F](javaEmil, upload),
             ScanMailboxTask.onCancel[F]
           )
         )
diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index 45dd1cf2..c016515b 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -6,6 +6,7 @@ import emil._
 //import emil.javamail.syntax._
 
 import docspell.common._
+import docspell.backend.ops.OUpload
 import docspell.store.records._
 import docspell.joex.scheduler.{Context, Task}
 
@@ -13,7 +14,7 @@ object ScanMailboxTask {
   val maxItems: Long = 7
   type Args = ScanMailboxArgs
 
-  def apply[F[_]: Sync](emil: Emil[F]): Task[F, Args, Unit] =
+  def apply[F[_]: Sync](emil: Emil[F], upload: OUpload[F]): Task[F, Args, Unit] =
     Task { ctx =>
       for {
         _ <- ctx.logger.info(
@@ -26,7 +27,7 @@ object ScanMailboxTask {
         _ <- ctx.logger.info(
           s"Reading mails for user ${userId.id} from ${imapConn.id}/${folders}"
         )
-        _ <- importMails(mailCfg, emil, ctx)
+        _ <- importMails(mailCfg, emil, upload, ctx)
       } yield ()
     }
 
@@ -49,7 +50,26 @@ object ScanMailboxTask {
   def importMails[F[_]: Sync](
       cfg: RUserImap,
       emil: Emil[F],
+      upload: OUpload[F],
       ctx: Context[F, Args]
   ): F[Unit] =
-    Sync[F].delay(println(s"$emil $ctx $cfg"))
+    Sync[F].delay(println(s"$emil $ctx $cfg $upload"))
+
+  object Impl {
+
+    // limit number of folders
+    // limit number of mails to retrieve per folder
+    // per folder:
+    //   fetch X mails
+    //     check via msgId if already present; if not:
+    //     load mail
+    //     serialize to *bytes*
+    //     store mail in queue
+    //     move mail or delete or do nothing
+    //     errors: log and keep going
+    //   errors per folder fetch: fail the task
+    //   notifiy joex after each batch
+    //
+    // no message id? make hash over complete mail or just import it
+  }
 }

From 451a09dda086706a9d058c310ea5520b6dee3a19 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Tue, 19 May 2020 05:47:14 +0200
Subject: [PATCH 08/14] Allow to skip joex notification on uploads

---
 .../scala/docspell/backend/ops/OUpload.scala  | 31 ++++++++++++++-----
 .../restserver/routes/UploadRoutes.scala      |  4 +--
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
index 3130e635..bc2c688e 100644
--- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
@@ -14,9 +14,17 @@ import org.log4s._
 
 trait OUpload[F[_]] {
 
-  def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult]
+  def submit(
+      data: OUpload.UploadData[F],
+      account: AccountId,
+      notifyJoex: Boolean
+  ): F[OUpload.UploadResult]
 
-  def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult]
+  def submit(
+      data: OUpload.UploadData[F],
+      sourceId: Ident,
+      notifyJoex: Boolean
+  ): F[OUpload.UploadResult]
 }
 
 object OUpload {
@@ -59,7 +67,8 @@ object OUpload {
 
       def submit(
           data: OUpload.UploadData[F],
-          account: AccountId
+          account: AccountId,
+          notifyJoex: Boolean
       ): F[OUpload.UploadResult] =
         for {
           files <- data.files.traverse(saveFile).map(_.flatten)
@@ -77,13 +86,17 @@ object OUpload {
             else Vector(ProcessItemArgs(meta, files.toList))
           job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
           _   <- logger.fdebug(s"Storing jobs: $job")
-          res <- job.traverse(submitJobs)
+          res <- job.traverse(submitJobs(notifyJoex))
           _ <- store.transact(
             RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)
           )
         } yield res.fold(identity, identity)
 
-      def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
+      def submit(
+          data: OUpload.UploadData[F],
+          sourceId: Ident,
+          notifyJoex: Boolean
+      ): F[OUpload.UploadResult] =
         for {
           sOpt <-
             store
@@ -92,14 +105,16 @@ object OUpload {
           abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
           updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
           accId  = sOpt.map(source => AccountId(source.cid, source.sid))
-          result <- accId.traverse(acc => submit(updata, acc))
+          result <- accId.traverse(acc => submit(updata, acc, notifyJoex))
         } yield result.fold(identity, identity)
 
-      private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] =
+      private def submitJobs(
+          notifyJoex: Boolean
+      )(jobs: Vector[RJob]): F[OUpload.UploadResult] =
         for {
           _ <- logger.fdebug(s"Storing jobs: $jobs")
           _ <- queue.insertAll(jobs)
-          _ <- joex.notifyAllNodes
+          _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F]
         } yield UploadResult.Success
 
       private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] =
diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
index bba60377..0689325f 100644
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
@@ -37,7 +37,7 @@ object UploadRoutes {
             Priority.High,
             cfg.backend.files.validMimeTypes
           )
-          result <- backend.upload.submit(updata, user.account)
+          result <- backend.upload.submit(updata, user.account, true)
           res    <- Ok(basicResult(result))
         } yield res
 
@@ -61,7 +61,7 @@ object UploadRoutes {
             Priority.Low,
             cfg.backend.files.validMimeTypes
           )
-          result <- backend.upload.submit(updata, id)
+          result <- backend.upload.submit(updata, id, true)
           res    <- Ok(basicResult(result))
         } yield res
 

From f2d67dc816ad2109b9912e9712f07334901fa06d Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Tue, 19 May 2020 07:39:02 +0200
Subject: [PATCH 09/14] Initial impl of import from mailbox user task

---
 .../main/scala/docspell/common/MimeType.scala |   1 +
 .../joex/src/main/resources/reference.conf    |  22 ++
 .../src/main/scala/docspell/joex/Config.scala |   4 +
 .../scala/docspell/joex/JoexAppImpl.scala     |   2 +-
 .../joex/scanmailbox/ScanMailboxTask.scala    | 224 +++++++++++++++++-
 .../scala/docspell/store/impl/Column.scala    |   4 +
 .../store/queries/QOrganization.scala         |  25 ++
 .../store/records/RAttachmentArchive.scala    |  21 ++
 8 files changed, 292 insertions(+), 11 deletions(-)

diff --git a/modules/common/src/main/scala/docspell/common/MimeType.scala b/modules/common/src/main/scala/docspell/common/MimeType.scala
index c312d370..f5230196 100644
--- a/modules/common/src/main/scala/docspell/common/MimeType.scala
+++ b/modules/common/src/main/scala/docspell/common/MimeType.scala
@@ -96,6 +96,7 @@ object MimeType {
   val tiff        = image("tiff")
   val html        = text("html")
   val plain       = text("plain")
+  val eml         = MimeType("message", "rfc822", Map.empty)
 
   object PdfMatch {
     def unapply(mt: MimeType): Option[MimeType] =
diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf
index fc9f7892..e4b1a38c 100644
--- a/modules/joex/src/main/resources/reference.conf
+++ b/modules/joex/src/main/resources/reference.conf
@@ -90,6 +90,28 @@ docspell.joex {
     wakeup-period = "10 minutes"
   }
 
+  # Configuration for the user-tasks.
+  user-tasks {
+    # Allows to import e-mails by scanning a mailbox.
+    scan-mailbox {
+      # A limit of how many folders to scan through. If a user
+      # configures more than this, only upto this limit folders are
+      # scanned and a warning is logged.
+      max-folders = 50
+
+      # How many mails (headers only) to retrieve in one chunk.
+      mail-chunk-size = 100
+
+      # A limit on how many mails to process in one job run. This is
+      # only to avoid resource allocation to one user/collective.
+      #
+      # If more than this number of mails is encountered, a warning is
+      # logged.
+      max-mails = 1000
+    }
+  }
+
+
   # Docspell uses periodic house keeping tasks, like cleaning expired
   # invites, that can be configured here.
   house-keeping {
diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala
index 58a9cb31..fb6df973 100644
--- a/modules/joex/src/main/scala/docspell/joex/Config.scala
+++ b/modules/joex/src/main/scala/docspell/joex/Config.scala
@@ -16,6 +16,7 @@ case class Config(
     jdbc: JdbcConfig,
     scheduler: SchedulerConfig,
     periodicScheduler: PeriodicSchedulerConfig,
+    userTasks: Config.UserTasks,
     houseKeeping: HouseKeepingConfig,
     extraction: ExtractConfig,
     textAnalysis: TextAnalysisConfig,
@@ -26,4 +27,7 @@ case class Config(
 
 object Config {
   case class Bind(address: String, port: Int)
+
+  case class ScanMailbox(maxFolders: Int, mailChunkSize: Int, maxMails: Int)
+  case class UserTasks(scanMailbox: ScanMailbox)
 }
diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
index 354c81cd..ea11a908 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
@@ -90,7 +90,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             ScanMailboxArgs.taskName,
-            ScanMailboxTask[F](javaEmil, upload),
+            ScanMailboxTask[F](cfg.userTasks.scanMailbox, javaEmil, upload, joex),
             ScanMailboxTask.onCancel[F]
           )
         )
diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index c016515b..e0956ef8 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -1,20 +1,32 @@
 package docspell.joex.scanmailbox
 
+import fs2._
 import cats.implicits._
 import cats.effect._
-import emil._
-//import emil.javamail.syntax._
+import emil.{MimeType => _, _}
+import emil.javamail.syntax._
+import emil.SearchQuery.{All, ReceivedDate}
 
 import docspell.common._
-import docspell.backend.ops.OUpload
+import docspell.backend.ops.{OJoex, OUpload}
 import docspell.store.records._
+import docspell.joex.Config
 import docspell.joex.scheduler.{Context, Task}
+import docspell.store.queries.QOrganization
+import cats.data.Kleisli
+import cats.data.NonEmptyList
+import cats.data.OptionT
 
 object ScanMailboxTask {
   val maxItems: Long = 7
   type Args = ScanMailboxArgs
 
-  def apply[F[_]: Sync](emil: Emil[F], upload: OUpload[F]): Task[F, Args, Unit] =
+  def apply[F[_]: Sync](
+      cfg: Config.ScanMailbox,
+      emil: Emil[F],
+      upload: OUpload[F],
+      joex: OJoex[F]
+  ): Task[F, Args, Unit] =
     Task { ctx =>
       for {
         _ <- ctx.logger.info(
@@ -27,7 +39,7 @@ object ScanMailboxTask {
         _ <- ctx.logger.info(
           s"Reading mails for user ${userId.id} from ${imapConn.id}/${folders}"
         )
-        _ <- importMails(mailCfg, emil, upload, ctx)
+        _ <- importMails(cfg, mailCfg, emil, upload, joex, ctx)
       } yield ()
     }
 
@@ -48,14 +60,206 @@ object ScanMailboxTask {
       }
 
   def importMails[F[_]: Sync](
-      cfg: RUserImap,
-      emil: Emil[F],
+      cfg: Config.ScanMailbox,
+      mailCfg: RUserImap,
+      theEmil: Emil[F],
       upload: OUpload[F],
+      joex: OJoex[F],
       ctx: Context[F, Args]
-  ): F[Unit] =
-    Sync[F].delay(println(s"$emil $ctx $cfg $upload"))
+  ): F[Unit] = {
+    val mailer    = theEmil(mailCfg.toMailConfig)
+    val impl      = new Impl[F](cfg, ctx)
+    val inFolders = ctx.args.folders.take(cfg.maxFolders)
 
-  object Impl {
+    val getInitialInput =
+      for {
+        _ <-
+          if (inFolders.size != ctx.args.folders.size)
+            ctx.logger.warn(
+              s"More than ${cfg.maxFolders} submitted. Only first ${cfg.maxFolders} will be scanned."
+            )
+          else ().pure[F]
+      } yield inFolders
+
+    def processFolders(in: Seq[String]): Stream[F, ScanResult] = {
+      val pass =
+        for {
+          name <- Stream.emits(in).covary[F]
+          res <-
+            Stream.eval(mailer.run(impl.handleFolder(theEmil.access, upload, joex)(name)))
+        } yield res
+
+      pass
+        .fold1(_ ++ _)
+        .flatMap { sr =>
+          if (sr.folders.isEmpty) Stream.emit(sr)
+          else processFolders(sr.folders)
+        }
+        .takeWhile(_.processed < cfg.maxMails)
+    }
+
+    Stream.eval(getInitialInput).flatMap(processFolders).compile.drain
+  }
+
+  case class ScanResult(folders: Seq[String], processed: Int, left: Int) {
+
+    /** Removes folders where nothing is left to process */
+    def ++(sr: ScanResult): ScanResult =
+      if (left == 0) ScanResult(sr.folders, processed + sr.processed, sr.left)
+      else if (sr.left == 0) ScanResult(folders, processed + sr.processed, left)
+      else ScanResult(folders ++ sr.folders, processed + sr.processed, left + sr.left)
+  }
+  object ScanResult {
+    def apply(folder: String, processed: Int, left: Int): ScanResult =
+      if (left <= 0) ScanResult(Seq.empty, processed, 0)
+      else ScanResult(Seq(folder), processed, left)
+  }
+
+  final private class Impl[F[_]: Sync](cfg: Config.ScanMailbox, ctx: Context[F, Args]) {
+
+    def handleFolder[C](a: Access[F, C], upload: OUpload[F], joex: OJoex[F])(
+        name: String
+    ): MailOp[F, C, ScanResult] =
+      for {
+        _       <- Kleisli.liftF(ctx.logger.info(s"Processing folder $name"))
+        folder  <- requireFolder(a)(name)
+        search  <- searchMails(a)(folder)
+        headers <- Kleisli.liftF(filterMessageIds(search.mails))
+        _       <- headers.traverse(handleOne(a, upload))
+        _       <- Kleisli.liftF(joex.notifyAllNodes)
+      } yield ScanResult(name, headers.size, search.count - search.mails.size)
+
+    def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] =
+      if ("INBOX".equalsIgnoreCase(name)) a.getInbox
+      else //TODO resolve sub-folders
+        a.findFolder(None, name)
+          .map(_.toRight(new Exception(s"Folder '$name' not found")))
+          .mapF(_.rethrow)
+
+    def searchMails[C](
+        a: Access[F, C]
+    )(folder: MailFolder): MailOp[F, C, SearchResult[MailHeader]] = {
+      val q = ctx.args.receivedSince match {
+        case Some(d) =>
+          Timestamp.current[F].map(now => ReceivedDate >= now.minus(d).value)
+        case None => All.pure[F]
+      }
+
+      for {
+        _ <- Kleisli.liftF(
+          ctx.logger.info(s"Searching next ${cfg.mailChunkSize} mails in ${folder.name}.")
+        )
+        query <- Kleisli.liftF(q)
+        mails <- a.search(folder, cfg.mailChunkSize)(query)
+      } yield mails
+    }
+
+    def filterMessageIds(headers: Vector[MailHeader]): F[Vector[MailHeader]] =
+      NonEmptyList.fromFoldable(headers.flatMap(_.messageId)) match {
+        case Some(nl) =>
+          for {
+            archives <- ctx.store.transact(
+              RAttachmentArchive
+                .findByMessageIdAndCollective(nl, ctx.args.account.collective)
+            )
+            existing = archives.flatMap(_.messageId).toSet
+            mails    = headers.filterNot(mh => mh.messageId.forall(existing.contains))
+            _ <- headers.size - mails.size match {
+              case 0 => ().pure[F]
+              case n => ctx.logger.info(s"Excluded $n mails since items for them already exist.")
+            }
+          } yield mails
+
+        case None =>
+          ctx.logger.info("No mails found") *> headers.pure[F]
+      }
+
+    def getDirection(mh: MailHeader): F[Direction] = {
+      val out: OptionT[F, Direction] =
+        for {
+          from <- OptionT.fromOption[F](mh.from)
+          _ <- OptionT(
+            ctx.store.transact(
+              QOrganization
+                .findPersonByContact(
+                  ctx.args.account.collective,
+                  from.address,
+                  Some(ContactKind.Email),
+                  Some(true)
+                )
+                .take(1)
+                .compile
+                .last
+            )
+          )
+        } yield Direction.Outgoing
+
+      OptionT
+        .fromOption[F](ctx.args.direction)
+        .orElse(out)
+        .getOrElse(Direction.Incoming)
+    }
+
+    def postHandle[C](a: Access[F, C])(mh: MailHeader): MailOp[F, C, Unit] =
+      ctx.args.targetFolder match {
+        case Some(tf) =>
+          a.getOrCreateFolder(None, tf).flatMap(folder => a.moveMail(mh, folder))
+        case None if ctx.args.deleteMail =>
+          a.deleteMail(mh).flatMapF { r =>
+            if (r.count == 0)
+              ctx.logger.warn(s"Mail '${mh.subject}' could not be deleted")
+            else ().pure[F]
+          }
+        case None =>
+          MailOp.pure(())
+      }
+
+    def submitMail(upload: OUpload[F])(mail: Mail[F]): F[OUpload.UploadResult] = {
+      val file = OUpload.File(
+        Some(mail.header.subject + ".eml"),
+        Some(MimeType.eml),
+        mail.toByteStream
+      )
+      for {
+        _   <- ctx.logger.debug(s"Submitting mail '${mail.header.subject}'")
+        dir <- getDirection(mail.header)
+        meta = OUpload.UploadMeta(
+          Some(dir),
+          s"mailbox-${ctx.args.account.user.id}",
+          Seq.empty
+        )
+        data = OUpload.UploadData(
+          multiple = false,
+          meta = meta,
+          files = Vector(file),
+          priority = Priority.Low,
+          tracker = None
+        )
+        res <- upload.submit(data, ctx.args.account, false)
+      } yield res
+    }
+
+    def handleOne[C](a: Access[F, C], upload: OUpload[F])(
+        mh: MailHeader
+    ): MailOp[F, C, Unit] =
+      for {
+        mail <- a.loadMail(mh)
+        res <- mail match {
+          case Some(m) =>
+            Kleisli.liftF(submitMail(upload)(m).attempt)
+          case None =>
+            MailOp.pure[F, C, Either[Throwable, OUpload.UploadResult]](
+              Either.left(new Exception(s"Mail not found"))
+            )
+        }
+        _ <- res.fold(
+          ex =>
+            Kleisli.liftF(
+              ctx.logger.warn(s"Error submitting '${mh.subject}': ${ex.getMessage}")
+            ),
+          _ => postHandle(a)(mh)
+        )
+      } yield ()
 
     // limit number of folders
     // limit number of mails to retrieve per folder
diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala
index c2f19abd..d84ed3cf 100644
--- a/modules/store/src/main/scala/docspell/store/impl/Column.scala
+++ b/modules/store/src/main/scala/docspell/store/impl/Column.scala
@@ -2,6 +2,7 @@ package docspell.store.impl
 
 import doobie._, doobie.implicits._
 import docspell.store.impl.DoobieSyntax._
+import cats.data.NonEmptyList
 
 case class Column(name: String, ns: String = "", alias: String = "") {
 
@@ -46,6 +47,9 @@ case class Column(name: String, ns: String = "", alias: String = "") {
   def isIn(values: Seq[Fragment]): Fragment =
     f ++ fr"IN (" ++ commas(values) ++ fr")"
 
+  def isIn[A: Put](values: NonEmptyList[A]): Fragment =
+    isIn(values.map(a => sql"$a").toList)
+
   def isIn(frag: Fragment): Fragment =
     f ++ fr"IN (" ++ frag ++ fr")"
 
diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala
index e9e00631..5fd63b15 100644
--- a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala
@@ -86,6 +86,31 @@ object QOrganization {
       })
   }
 
+  def findPersonByContact(
+      coll: Ident,
+      value: String,
+      ck: Option[ContactKind],
+      concerning: Option[Boolean]
+  ): Stream[ConnectionIO, RPerson] = {
+    val pColl = PC.cid.prefix("p")
+    val pConc = PC.concerning.prefix("p")
+    val pId   = PC.pid.prefix("p")
+    val cPers = RContact.Columns.personId.prefix("c")
+    val cVal  = RContact.Columns.value.prefix("c")
+    val cKind = RContact.Columns.kind.prefix("c")
+
+    val from = RPerson.table ++ fr"p INNER JOIN" ++
+      RContact.table ++ fr"c ON" ++ cPers.is(pId)
+    val q = Seq(
+      cVal.lowerLike(s"%${value.toLowerCase}%"),
+      pColl.is(coll)
+    ) ++ concerning.map(pConc.is(_)).toSeq ++ ck.map(cKind.is(_)).toSeq
+
+    selectDistinct(PC.all.map(_.prefix("p")), from, and(q))
+      .query[RPerson]
+      .stream
+  }
+
   def addOrg[F[_]](
       org: ROrganization,
       contacts: Seq[RContact],
diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala
index 04dd38b0..09ccbab7 100644
--- a/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentArchive.scala
@@ -6,6 +6,7 @@ import doobie.implicits._
 import docspell.common._
 import docspell.store.impl._
 import docspell.store.impl.Implicits._
+import cats.data.NonEmptyList
 
 /** The archive file of some attachment. The `id` is shared with the
   * attachment, to create a 0..1-1 relationship.
@@ -72,6 +73,26 @@ object RAttachmentArchive {
     selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].option
   }
 
+  def findByMessageIdAndCollective(
+      messageIds: NonEmptyList[String],
+      collective: Ident
+  ): ConnectionIO[Vector[RAttachmentArchive]] = {
+    val bId    = RAttachment.Columns.id.prefix("b")
+    val bItem  = RAttachment.Columns.itemId.prefix("b")
+    val aMsgId = Columns.messageId.prefix("a")
+    val aId    = Columns.id.prefix("a")
+    val iId    = RItem.Columns.id.prefix("i")
+    val iColl  = RItem.Columns.cid.prefix("i")
+
+    val from = table ++ fr"a INNER JOIN" ++
+      RAttachment.table ++ fr"b ON" ++ aId.is(bId) ++
+      fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ bItem.is(iId)
+
+    val where = and(aMsgId.isIn(messageIds), iColl.is(collective))
+
+    selectSimple(all.map(_.prefix("a")), from, where).query[RAttachmentArchive].to[Vector]
+  }
+
   def findByItemWithMeta(
       id: Ident
   ): ConnectionIO[Vector[(RAttachmentArchive, FileMeta)]] = {

From 31a1abf3950212fa78dd0f20aab6b52285050f90 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Wed, 20 May 2020 00:17:52 +0200
Subject: [PATCH 10/14] Add server limits to importing mails task

---
 .../joex/scanmailbox/ScanMailboxTask.scala    | 97 ++++++++++---------
 1 file changed, 53 insertions(+), 44 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index e0956ef8..572f46a8 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -81,38 +81,54 @@ object ScanMailboxTask {
           else ().pure[F]
       } yield inFolders
 
-    def processFolders(in: Seq[String]): Stream[F, ScanResult] = {
-      val pass =
-        for {
-          name <- Stream.emits(in).covary[F]
-          res <-
-            Stream.eval(mailer.run(impl.handleFolder(theEmil.access, upload, joex)(name)))
-        } yield res
+    def processFolder(acc: ScanResult, name: String): F[ScanResult] =
+      if (acc.noneLeft(name)) acc.pure[F]
+      else
+        mailer
+          .run(impl.handleFolder(theEmil.access, upload, joex)(name))
+          .map(_ ++ acc)
 
-      pass
-        .fold1(_ ++ _)
-        .flatMap { sr =>
-          if (sr.folders.isEmpty) Stream.emit(sr)
-          else processFolders(sr.folders)
-        }
-        .takeWhile(_.processed < cfg.maxMails)
+    Stream
+      .eval(getInitialInput)
+      .flatMap(Stream.emits)
+      .repeat
+      .evalScan(ScanResult.empty)(processFolder)
+      .takeThrough(result =>
+        result.processed < cfg.maxMails && result.someLeft(inFolders.size)
+      )
+      .lastOr(ScanResult.empty)
+      .evalMap { sr =>
+        if (sr.processed >= cfg.maxMails)
+          ctx.logger.warn(
+            s"Reached server maximum of ${cfg.maxMails} processed mails. Processed ${sr.processed} mails."
+          )
+        else
+          ctx.logger
+            .info(s"Stopped after processing ${sr.processed} mails")
+      }
+      .compile
+      .drain
+  }
+
+  case class ScanResult(folders: List[(String, Int)], processed: Int) {
+
+    def ++(sr: ScanResult): ScanResult = {
+      val fs = (folders ++ sr.folders).sortBy(_._2).distinctBy(_._1)
+      ScanResult(fs, processed + sr.processed)
     }
 
-    Stream.eval(getInitialInput).flatMap(processFolders).compile.drain
+    def noneLeft(name: String): Boolean =
+      folders.find(_._1 == name).exists(_._2 <= 0)
+
+    def someLeft(inputFolders: Int) =
+      ScanResult.empty == this || folders.exists(_._2 > 0) || inputFolders > folders.size
+
   }
 
-  case class ScanResult(folders: Seq[String], processed: Int, left: Int) {
-
-    /** Removes folders where nothing is left to process */
-    def ++(sr: ScanResult): ScanResult =
-      if (left == 0) ScanResult(sr.folders, processed + sr.processed, sr.left)
-      else if (sr.left == 0) ScanResult(folders, processed + sr.processed, left)
-      else ScanResult(folders ++ sr.folders, processed + sr.processed, left + sr.left)
-  }
   object ScanResult {
+    val empty = ScanResult(Nil, 0)
     def apply(folder: String, processed: Int, left: Int): ScanResult =
-      if (left <= 0) ScanResult(Seq.empty, processed, 0)
-      else ScanResult(Seq(folder), processed, left)
+      ScanResult(List(folder -> left), processed)
   }
 
   final private class Impl[F[_]: Sync](cfg: Config.ScanMailbox, ctx: Context[F, Args]) {
@@ -127,7 +143,7 @@ object ScanMailboxTask {
         headers <- Kleisli.liftF(filterMessageIds(search.mails))
         _       <- headers.traverse(handleOne(a, upload))
         _       <- Kleisli.liftF(joex.notifyAllNodes)
-      } yield ScanResult(name, headers.size, search.count - search.mails.size)
+      } yield ScanResult(name, search.mails.size, search.count - search.mails.size)
 
     def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] =
       if ("INBOX".equalsIgnoreCase(name)) a.getInbox
@@ -147,10 +163,17 @@ object ScanMailboxTask {
 
       for {
         _ <- Kleisli.liftF(
-          ctx.logger.info(s"Searching next ${cfg.mailChunkSize} mails in ${folder.name}.")
+          ctx.logger.debug(
+            s"Searching next ${cfg.mailChunkSize} mails in ${folder.name}."
+          )
         )
         query <- Kleisli.liftF(q)
         mails <- a.search(folder, cfg.mailChunkSize)(query)
+        _ <- Kleisli.liftF(
+          ctx.logger.debug(
+            s"Found ${mails.count} mails in folder. Reading first ${mails.mails.size}"
+          )
+        )
       } yield mails
     }
 
@@ -166,12 +189,13 @@ object ScanMailboxTask {
             mails    = headers.filterNot(mh => mh.messageId.forall(existing.contains))
             _ <- headers.size - mails.size match {
               case 0 => ().pure[F]
-              case n => ctx.logger.info(s"Excluded $n mails since items for them already exist.")
+              case n =>
+                ctx.logger.info(s"Excluded $n mails since items for them already exist.")
             }
           } yield mails
 
         case None =>
-          ctx.logger.info("No mails found") *> headers.pure[F]
+          headers.pure[F]
       }
 
     def getDirection(mh: MailHeader): F[Direction] = {
@@ -260,20 +284,5 @@ object ScanMailboxTask {
           _ => postHandle(a)(mh)
         )
       } yield ()
-
-    // limit number of folders
-    // limit number of mails to retrieve per folder
-    // per folder:
-    //   fetch X mails
-    //     check via msgId if already present; if not:
-    //     load mail
-    //     serialize to *bytes*
-    //     store mail in queue
-    //     move mail or delete or do nothing
-    //     errors: log and keep going
-    //   errors per folder fetch: fail the task
-    //   notifiy joex after each batch
-    //
-    // no message id? make hash over complete mail or just import it
   }
 }

From 2858d6b8535604e95aee0f34ca7cb64c086d05a4 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Wed, 20 May 2020 19:44:45 +0200
Subject: [PATCH 11/14] Notify job executors at the end of the task

---
 .../joex/scanmailbox/ScanMailboxTask.scala    | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index 572f46a8..8af6b6a7 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -85,7 +85,7 @@ object ScanMailboxTask {
       if (acc.noneLeft(name)) acc.pure[F]
       else
         mailer
-          .run(impl.handleFolder(theEmil.access, upload, joex)(name))
+          .run(impl.handleFolder(theEmil.access, upload)(name))
           .map(_ ++ acc)
 
     Stream
@@ -98,13 +98,14 @@ object ScanMailboxTask {
       )
       .lastOr(ScanResult.empty)
       .evalMap { sr =>
-        if (sr.processed >= cfg.maxMails)
-          ctx.logger.warn(
-            s"Reached server maximum of ${cfg.maxMails} processed mails. Processed ${sr.processed} mails."
-          )
-        else
-          ctx.logger
-            .info(s"Stopped after processing ${sr.processed} mails")
+        joex.notifyAllNodes *>
+          (if (sr.processed >= cfg.maxMails)
+             ctx.logger.warn(
+               s"Reached server maximum of ${cfg.maxMails} processed mails. Processed ${sr.processed} mails."
+             )
+           else
+             ctx.logger
+               .info(s"Stopped after processing ${sr.processed} mails"))
       }
       .compile
       .drain
@@ -133,7 +134,7 @@ object ScanMailboxTask {
 
   final private class Impl[F[_]: Sync](cfg: Config.ScanMailbox, ctx: Context[F, Args]) {
 
-    def handleFolder[C](a: Access[F, C], upload: OUpload[F], joex: OJoex[F])(
+    def handleFolder[C](a: Access[F, C], upload: OUpload[F])(
         name: String
     ): MailOp[F, C, ScanResult] =
       for {
@@ -142,7 +143,6 @@ object ScanMailboxTask {
         search  <- searchMails(a)(folder)
         headers <- Kleisli.liftF(filterMessageIds(search.mails))
         _       <- headers.traverse(handleOne(a, upload))
-        _       <- Kleisli.liftF(joex.notifyAllNodes)
       } yield ScanResult(name, search.mails.size, search.count - search.mails.size)
 
     def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] =

From c0259dba7e8338d3867ab6d3994f7d04523b57b6 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Wed, 20 May 2020 22:15:25 +0200
Subject: [PATCH 12/14] Allow to enable debug flag for javamail

---
 modules/joex/src/main/resources/reference.conf              | 6 ++++++
 modules/joex/src/main/scala/docspell/joex/Config.scala      | 3 ++-
 modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala | 3 ++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf
index e4b1a38c..a9b76e71 100644
--- a/modules/joex/src/main/resources/reference.conf
+++ b/modules/joex/src/main/resources/reference.conf
@@ -31,6 +31,12 @@ docspell.joex {
     password = ""
   }
 
+  # Enable or disable debugging for e-mail related functionality. This
+  # applies to both sending and receiving mails. For security reasons
+  # logging is not very extensive on authentication failures. Setting
+  # this to true, results in a lot of data printed to stdout.
+  mail-debug = false
+
   send-mail {
     # This is used as the List-Id e-mail header when mails are sent
     # from docspell to its users (example: for notification mails). It
diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala
index fb6df973..7292e5f9 100644
--- a/modules/joex/src/main/scala/docspell/joex/Config.scala
+++ b/modules/joex/src/main/scala/docspell/joex/Config.scala
@@ -22,7 +22,8 @@ case class Config(
     textAnalysis: TextAnalysisConfig,
     convert: ConvertConfig,
     sendMail: MailSendConfig,
-    files: Files
+    files: Files,
+    mailDebug: Boolean
 )
 
 object Config {
diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
index ea11a908..d07ca841 100644
--- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
+++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
@@ -70,7 +70,8 @@ object JoexAppImpl {
       nodeOps <- ONode(store)
       joex    <- OJoex(client, store)
       upload  <- OUpload(store, queue, cfg.files, joex)
-      javaEmil = JavaMailEmil(blocker)
+      javaEmil =
+        JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))
       sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
         .withQueue(queue)
         .withTask(

From d9782582d881e7e9113dd8ab5e043fdfebc8b554 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Wed, 20 May 2020 22:44:29 +0200
Subject: [PATCH 13/14] Use `max-mails` setting with higher priority

The `mail-chunk-size` is set to its configured value or `max-mails`
whichever is lower.
---
 modules/joex/src/main/resources/reference.conf         | 10 +++++++---
 modules/joex/src/main/scala/docspell/joex/Config.scala |  5 ++++-
 .../docspell/joex/scanmailbox/ScanMailboxTask.scala    |  4 ++--
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf
index a9b76e71..1958070b 100644
--- a/modules/joex/src/main/resources/reference.conf
+++ b/modules/joex/src/main/resources/reference.conf
@@ -106,14 +106,18 @@ docspell.joex {
       max-folders = 50
 
       # How many mails (headers only) to retrieve in one chunk.
-      mail-chunk-size = 100
+      #
+      # If this is greater than `max-mails' it is set automatically to
+      # the value of `max-mails'.
+      mail-chunk-size = 50
 
       # A limit on how many mails to process in one job run. This is
-      # only to avoid resource allocation to one user/collective.
+      # meant to avoid too heavy resource allocation to one
+      # user/collective.
       #
       # If more than this number of mails is encountered, a warning is
       # logged.
-      max-mails = 1000
+      max-mails = 500
     }
   }
 
diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala
index 7292e5f9..27817d69 100644
--- a/modules/joex/src/main/scala/docspell/joex/Config.scala
+++ b/modules/joex/src/main/scala/docspell/joex/Config.scala
@@ -29,6 +29,9 @@ case class Config(
 object Config {
   case class Bind(address: String, port: Int)
 
-  case class ScanMailbox(maxFolders: Int, mailChunkSize: Int, maxMails: Int)
+  case class ScanMailbox(maxFolders: Int, mailChunkSize: Int, maxMails: Int) {
+    def mailBatchSize: Int =
+      math.min(mailChunkSize, maxMails)
+  }
   case class UserTasks(scanMailbox: ScanMailbox)
 }
diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
index 8af6b6a7..670b2fec 100644
--- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
+++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala
@@ -164,11 +164,11 @@ object ScanMailboxTask {
       for {
         _ <- Kleisli.liftF(
           ctx.logger.debug(
-            s"Searching next ${cfg.mailChunkSize} mails in ${folder.name}."
+            s"Searching next ${cfg.mailBatchSize} mails in ${folder.name}."
           )
         )
         query <- Kleisli.liftF(q)
-        mails <- a.search(folder, cfg.mailChunkSize)(query)
+        mails <- a.search(folder, cfg.mailBatchSize)(query)
         _ <- Kleisli.liftF(
           ctx.logger.debug(
             s"Found ${mails.count} mails in folder. Reading first ${mails.mails.size}"

From 09b6b1bb05550bab5193f8e24791877f85ed6a59 Mon Sep 17 00:00:00 2001
From: Eike Kettner <eike.kettner@posteo.de>
Date: Wed, 20 May 2020 22:58:19 +0200
Subject: [PATCH 14/14] Improve user-task texts

---
 .../src/main/elm/Page/UserSettings/View.elm   | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
index 0ea0a609..14a69583 100644
--- a/modules/webapp/src/main/elm/Page/UserSettings/View.elm
+++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm
@@ -111,14 +111,15 @@ viewNotificationForm model =
             ]
         ]
     , p []
-        [ text "Docspell can notify you once the due dates of your items come closer. "
-        , text "Notification is done via e-mail. You need to provide a connection in "
-        , text "your e-mail settings."
+        [ text """
+            Docspell can notify you once the due dates of your items
+            come closer. Notification is done via e-mail. You need to
+            provide a connection in your e-mail settings."""
         ]
     , p []
-        [ text "Each time this is executed, docspell finds all items that are due in "
+        [ text "Docspell finds all items that are due in "
         , em [] [ text "Remind Days" ]
-        , text " days."
+        , text " days and sends this list via e-mail."
         ]
     , Html.map NotificationMsg
         (Comp.NotificationForm.view "segment" model.notificationModel)
@@ -134,14 +135,20 @@ viewScanMailboxForm model =
             ]
         ]
     , p []
-        [ text "Docspell can scan folders of your mailbox for mails to import. "
+        [ text "Docspell can scan folders of your mailbox to import your mails. "
         , text "You need to provide a connection in "
         , text "your e-mail (imap) settings."
         ]
     , p []
-        [ text "Each time this is executed, docspell goes through all configured folders "
-        , text "and imports mails matching the search criteria. The number of mails to import "
-        , text "at one task run is limited. Mails already read in are skipped."
+        [ text """
+            Docspell goes through all configured folders and imports
+            mails matching the search criteria. Mails are skipped if
+            they were imported in a previous run and the corresponding
+            items still exist. After submitting a mail into docspell,
+            you can choose to move it to another folder, to delete it
+            or to just leave it there. In the latter case you should
+            adjust the schedule to avoid reading over the same mails
+            again."""
         ]
     , Html.map ScanMailboxMsg
         (Comp.ScanMailboxForm.view "segment" model.scanMailboxModel)