Merge branch 'master' into update/flexmark-0.61.24

Changelog.md

@@ -2,19 +2,28 @@
 
 ## v0.5.0
 
-*Unknown*
+*May 1st, 2020*
 
 - Allow to delete attachments of an item.
 - Allow to be notified via e-mail for items with a due date. This uses
   the periodic-task framework introduced in the last release.
 - Fix issues when converting HTML with unkown links. This especially
   happens with e-mails that contain images to attachments.
-- Fix issues when importing e-mail files:
+- Fix various issues when importing e-mail files, for example:
   - fixes encoding problems for mails without explicit transfer encoding
   - add meta info (from, to, subject) to the converted pdf document
   - clean html mails to remove unwanted content (like javascript)
 - Fix classpath issue with javax.mail vs jakarta.mail
 
+### Configuration Changes
+
+The Joex component has config changes:
+
+- A new section `send-mail` containing a `List-Id` e-mail header to
+  use. Use an empty string (the default) to avoid setting such header.
+  This header is only applied for notification mails.
+
+
 ## v0.4.0 
 
 *Mar. 29, 2020*

build.sbt

@@ -141,6 +141,9 @@ val openapiScalaSettings = Seq(
 
 // --- Modules
 
+// Base module, everything depends on this – including restapi and
+// joexapi modules. This should aim to have least possible
+// dependencies
 val common = project.in(file("modules/common")).
   disablePlugins(RevolverPlugin).
   settings(sharedSettings).

docker/build-images.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Update the versions in joex.dockerfile and restserver.dockerfile,
+# docker-compose.yml and joex/entrypoint.sh; update versions here
+
+docker build -t eikek0/docspell:joex-0.5.0 -f joex.dockerfile  .
+docker build -t eikek0/docspell:restserver-0.5.0 -f restserver.dockerfile  .
+
+# test with docker-compose up

docker/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   restserver:
-    image: eikek0/docspell:restserver-0.4.0
+    image: eikek0/docspell:restserver-0.5.0
     container_name: docspell-restserver
     command: /opt/docspell.conf
     ports:
@@ -20,7 +20,7 @@ services:
       - POSTGRES_PASSWORD=dbpass
       - POSTGRES_DB=dbname
   joex:
-    image: eikek0/docspell:joex-0.4.0
+    image: eikek0/docspell:joex-0.5.0
     container_name: docspell-joex
     command: /opt/docspell.conf
     ports:

docker/joex.dockerfile

@@ -24,7 +24,7 @@ RUN apk add --no-cache openjdk11-jre \
   && ln -s /usr/bin/python3 /usr/bin/python \
   && mkdir -p /opt \
   && cd /opt \
-  && curl -L -o docspell.zip https://github.com/eikek/docspell/releases/download/v0.4.0/docspell-joex-0.4.0.zip \
+  && curl -L -o docspell.zip https://github.com/eikek/docspell/releases/download/v0.5.0/docspell-joex-0.5.0.zip \
   && unzip docspell.zip \
   && rm docspell.zip \
   && apk del curl unzip

docker/joex/entrypoint.sh

@@ -3,4 +3,4 @@
 echo "Starting unoconv listener"
 unoconv -l &
 
-/opt/docspell-joex-0.4.0/bin/docspell-joex "$@"
+/opt/docspell-joex-0.5.0/bin/docspell-joex "$@"

docker/restserver.dockerfile

@@ -6,11 +6,11 @@ RUN apk add --no-cache openjdk11-jre unzip curl bash
 
 RUN mkdir -p /opt \
   && cd /opt \
-  && curl -L -o docspell.zip https://github.com/eikek/docspell/releases/download/v0.4.0/docspell-restserver-0.4.0.zip \
+  && curl -L -o docspell.zip https://github.com/eikek/docspell/releases/download/v0.5.0/docspell-restserver-0.5.0.zip \
   && unzip docspell.zip \
   && rm docspell.zip \
   && apk del unzip curl
 
 EXPOSE 7880
 
-ENTRYPOINT ["/opt/docspell-restserver-0.4.0/bin/docspell-restserver"]
+ENTRYPOINT ["/opt/docspell-restserver-0.5.0/bin/docspell-restserver"]

modules/backend/src/main/scala/docspell/backend/ops/OMail.scala

@@ -158,6 +158,7 @@ object OMail {
             val fields: Seq[Trans[F]] = Seq(
               From(sett.mailFrom),
               Tos(m.recipients),
+              XMailer.emil,
               Subject(m.subject),
               TextBody[F](m.body)
             )

modules/common/src/main/scala/docspell/common/MailSendConfig.scala

@@ -0,0 +1,3 @@
+package docspell.common
+
+case class MailSendConfig(listId: String)

modules/joex/src/main/resources/reference.conf

@@ -31,6 +31,19 @@ docspell.joex {
     password = ""
   }
 
+  send-mail {
+    # This is used as the List-Id e-mail header when mails are sent
+    # from docspell to its users (example: for notification mails). It
+    # is not used when sending to external recipients. If it is empty,
+    # no such header is added. Using this header is often useful when
+    # filtering mails.
+    #
+    # It should be a string in angle brackets. See
+    # https://tools.ietf.org/html/rfc2919 for a formal specification
+    # of this header.
+    list-id = ""
+  }
+
   # Configuration for the job scheduler.
   scheduler {
 

modules/joex/src/main/scala/docspell/joex/Config.scala

@@ -1,7 +1,7 @@
 package docspell.joex
 
 import docspell.analysis.TextAnalysisConfig
-import docspell.common.{Ident, LenientUri}
+import docspell.common._
 import docspell.joex.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
 import docspell.store.JdbcConfig
 import docspell.convert.ConvertConfig
@@ -18,7 +18,8 @@ case class Config(
     houseKeeping: HouseKeepingConfig,
     extraction: ExtractConfig,
     textAnalysis: TextAnalysisConfig,
-    convert: ConvertConfig
+    convert: ConvertConfig,
+    sendMail: MailSendConfig
 )
 
 object Config {

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -78,7 +78,7 @@ object JoexAppImpl {
         .withTask(
           JobTask.json(
             NotifyDueItemsArgs.taskName,
-            NotifyDueItemsTask[F](JavaMailEmil(blocker)),
+            NotifyDueItemsTask[F](cfg.sendMail, JavaMailEmil(blocker)),
             NotifyDueItemsTask.onCancel[F]
           )
         )

modules/joex/src/main/scala/docspell/joex/mail/EmilHeader.scala

@@ -0,0 +1,14 @@
+package docspell.joex.mail
+
+import emil.builder._
+
+object EmilHeader {
+
+  // Remove with next emil version
+  def optionalHeader[F[_]](name: String, value: Option[String]): Trans[F] =
+    value.map(v => CustomHeader[F](name, v)).getOrElse(Trans[F](identity))
+
+  def listId[F[_]](listId: String): Trans[F] =
+    optionalHeader("List-Id", Option(listId).filter(_.nonEmpty))
+
+}

modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala

@@ -12,6 +12,7 @@ import docspell.store.records._
 import docspell.store.queries.QItem
 import docspell.joex.scheduler.{Context, Task}
 import cats.data.OptionT
+import docspell.joex.mail.EmilHeader
 import docspell.joex.notify.MailContext
 import docspell.joex.notify.MailTemplate
 
@@ -19,7 +20,7 @@ object NotifyDueItemsTask {
   val maxItems: Long = 7
   type Args = NotifyDueItemsArgs
 
-  def apply[F[_]: Sync](emil: Emil[F]): Task[F, Args, Unit] =
+  def apply[F[_]: Sync](cfg: MailSendConfig, emil: Emil[F]): Task[F, Args, Unit] =
     Task { ctx =>
       for {
         _       <- ctx.logger.info("Getting mail configuration")
@@ -27,7 +28,7 @@ object NotifyDueItemsTask {
         _ <- ctx.logger.info(
           s"Searching for items due in ${ctx.args.remindDays} days…."
         )
-        _ <- createMail(mailCfg, ctx)
+        _ <- createMail(cfg, mailCfg, ctx)
           .semiflatMap { mail =>
             for {
               _   <- ctx.logger.info(s"Sending notification mail to ${ctx.args.recipients}")
@@ -56,12 +57,13 @@ object NotifyDueItemsTask {
       }
 
   def createMail[F[_]: Sync](
+      sendCfg: MailSendConfig,
       cfg: RUserEmail,
       ctx: Context[F, Args]
   ): OptionT[F, Mail[F]] =
     for {
       items <- OptionT.liftF(findItems(ctx)).filter(_.nonEmpty)
-      mail  <- OptionT.liftF(makeMail(cfg, ctx.args, items))
+      mail  <- OptionT.liftF(makeMail(sendCfg, cfg, ctx.args, items))
     } yield mail
 
   def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[QItem.ListItem]] =
@@ -81,6 +83,7 @@ object NotifyDueItemsTask {
     } yield res
 
   def makeMail[F[_]: Sync](
+      sendCfg: MailSendConfig,
       cfg: RUserEmail,
       args: Args,
       items: Vector[QItem.ListItem]
@@ -99,7 +102,9 @@ object NotifyDueItemsTask {
       MailBuilder.build(
         From(cfg.mailFrom),
         Tos(recp),
-        Subject("Next due items"),
+        XMailer.emil,
+        Subject("[Docspell] Next due items"),
+        EmilHeader.listId(sendCfg.listId),
         MarkdownBody[F](md).withConfig(
           MarkdownConfig("body { font-size: 10pt; font-family: sans-serif; }")
         )

modules/joexapi/src/main/resources/joex-openapi.yml

@@ -2,7 +2,7 @@ openapi: 3.0.0
 
 info:
   title: Docspell JOEX
-  version: 0.5.0-SNAPSHOT
+  version: 0.6.0-SNAPSHOT
 
 servers:
   - url: /api/v1

modules/microsite/docs/features.md

@@ -20,6 +20,7 @@ permalink: features
 - Web-UI included
 - Create “share-urls” to upload files anonymously
 - Send documents via e-mail
+- E-Mail notification for documents with due dates
 - REST server and document processing are separate applications which
   can be scaled-out independently
 - Everything stored in a SQL database: PostgreSQL, MariaDB or H2

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -2,7 +2,7 @@ openapi: 3.0.0
 
 info:
   title: Docspell
-  version: 0.5.0-SNAPSHOT
+  version: 0.6.0-SNAPSHOT
   description: |
     This is the remote API to Docspell, a personal document organizer.
 

modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala

@@ -18,7 +18,7 @@ import docspell.store.queries.QUserTask
   * `RPeriodicTask`. A user task is associated to a specific user (not
   * just the collective).
   *
-  * @implNote: The mapping is as follows: The collective is the task
+  * implNote: The mapping is as follows: The collective is the task
   * group. The submitter property contains the username. Once a task
   * is saved to the database, it can only be refernced uniquely by its
   * id. A user may submit multiple same tasks (with different

modules/webapp/src/main/elm/Comp/ItemDetail.elm

@@ -1033,7 +1033,7 @@ view inav model =
                     [ ( "toggle item", True )
                     , ( "active", model.menuOpen )
                     ]
-                , title "Edit item"
+                , title "Edit Metadata"
                 , onClick ToggleMenu
                 , href ""
                 ]
@@ -1050,6 +1050,21 @@ view inav model =
                 ]
                 [ i [ class "mail outline icon" ] []
                 ]
+            , a
+                [ classList
+                    [ ( "toggle item", True )
+                    , ( "active", isEditNotes model.notesField )
+                    ]
+                , if isEditNotes model.notesField then
+                    title "Cancel editing"
+
+                  else
+                    title "Edit Notes"
+                , onClick ToggleEditNotes
+                , href "#"
+                ]
+                [ i [ class "edit outline icon" ] []
+                ]
             ]
         , renderMailForm model
         , div [ class "ui grid" ]
@@ -1137,13 +1152,6 @@ renderNotes model =
                             ]
                             [ i [ class "eye slash icon" ] []
                             ]
-                        , a
-                            [ class "ui right corner label"
-                            , onClick ToggleEditNotes
-                            , href "#"
-                            ]
-                            [ i [ class "edit icon" ] []
-                            ]
                         ]
                     ]
 
@@ -1545,16 +1553,6 @@ renderEditButtons model =
             [ i [ class "trash icon" ] []
             , text "Delete"
             ]
-        , button
-            [ classList
-                [ ( "ui secondary right floated icon button", True )
-                , ( "basic", model.notesField == HideNotes || model.notesField == ViewNotes )
-                ]
-            , title "Toggle Notes Form"
-            , onClick ToggleEditNotes
-            ]
-            [ i [ class "edit outline icon" ] []
-            ]
         ]
 
 

nix/buildvm.sh

@@ -7,4 +7,4 @@ fi
 
 nixos-rebuild build-vm \
   -I nixos-config=./configuration-test.nix \
-  -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz
+  -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz

nix/configuration-test.nix

@@ -6,9 +6,9 @@ in
   imports = docspell.modules;
 
   i18n = {
-    consoleKeyMap = "neo";
     defaultLocale = "en_US.UTF-8";
   };
+  console.keyMap = "neo";
 
   users.users.root = {
     password = "root";
@@ -56,6 +56,6 @@ in
     firewall.allowedTCPPorts = [7880];
   };
 
-  system.stateVersion = "19.09";
+  system.stateVersion = "20.03";
 
 }

nix/module-joex.nix

@@ -21,6 +21,9 @@ let
       user = "sa";
       password = "";
     };
+    send-mail = {
+      list-id = "";
+    };
     scheduler = {
       pool-size = 2;
       counting-scheme = "4,1";
@@ -202,6 +205,30 @@ in {
         description = "Database connection settings";
       };
 
+      send-mail = mkOption {
+        type = types.submodule({
+          options = {
+            list-id = mkOption {
+              type = types.str;
+              default = defaults.send-mail.list-id;
+              description = ''
+                This is used as the List-Id e-mail header when mails are sent
+                from docspell to its users (example: for notification mails). It
+                is not used when sending to external recipients. If it is empty,
+                no such header is added. Using this header is often useful when
+                filtering mails.
+
+                It should be a string in angle brackets. See
+                https://tools.ietf.org/html/rfc2919 for a formal specification
+              '';
+            };
+
+          };
+        });
+        default = defaults.send-mail;
+        description = "Settings for sending mails.";
+      };
+
       scheduler = mkOption {
         type = types.submodule({
           options = {

nix/pkg.nix

@@ -10,7 +10,7 @@ in
 { server = stdenv.mkDerivation rec {
     name = "docspell-server-${cfg.version}";
 
-     src = fetchzip cfg.server;
+    src = fetchzip cfg.server;
 
     buildInputs = [ jre8 ];
 

nix/release.nix

@@ -1,5 +1,20 @@
 rec {
   cfg = {
+    v0_5_0 = rec {
+      version = "0.5.0";
+      server = {
+        url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-restserver-${version}.zip";
+        sha256 = "1cr1x5ncl8prrp50mip17filyh2g1hq4ycjq4h4zmaj1nlvzrfy5";
+      };
+      joex = {
+        url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-joex-${version}.zip";
+        sha256 = "1pgkay99h59c2hnxhibrg8dy2j5bmlkv1hi18snccf7d304xl6w6";
+      };
+      tools = {
+        url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-tools-${version}.zip";
+        sha256 = "1bqm3bwg4n2llbsipp9ydmlkk3hv0x0jx482x4jb98x0fjyipzyy";
+      };
+    };
     v0_4_0 = rec {
       version = "0.4.0";
       server = {
@@ -62,7 +77,7 @@ rec {
     };
   };
   pkg = v: import ./pkg.nix v;
-  currentPkg = pkg cfg.v0_4_0;
+  currentPkg = pkg cfg.v0_5_0;
   module-joex = ./module-joex.nix;
   module-restserver = ./module-server.nix;
   module-consumedir = ./module-consumedir.nix;

project/Dependencies.scala

@@ -12,8 +12,8 @@ object Dependencies {
   val DoobieVersion = "0.9.0"
   val EmilVersion = "0.5.1"
   val FastparseVersion = "2.1.3"
-  val FlexmarkVersion = "0.61.24"
+  val FlexmarkVersion = "0.61.20"
-  val FlywayVersion = "6.4.0"
+  val FlywayVersion = "6.4.1"
   val Fs2Version = "2.3.0"
   val H2Version = "1.4.200"
   val Http4sVersion = "0.21.4"

version.sbt

@@ -1 +1 @@
-version in ThisBuild := "0.5.0-SNAPSHOT"
+version in ThisBuild := "0.6.0-SNAPSHOT"