Add docs for how to create a smtp gateway

build.sbt

@@ -432,21 +432,27 @@ val microsite = project.in(file("modules/microsite")).
       )
     ),
     Compile/resourceGenerators += Def.task {
-      val conf1 = (resourceDirectory in (restserver, Compile)).value / "reference.conf"
-      val conf2 = (resourceDirectory in (joex, Compile)).value / "reference.conf"      
-      val out1 = resourceManaged.value/"main"/"jekyll"/"_includes"/"server.conf"
-      val out2 = resourceManaged.value/"main"/"jekyll"/"_includes"/"joex.conf"
-      streams.value.log.info(s"Copying reference.conf: $conf1 -> $out1, $conf2 -> $out2")
-      IO.write(out1, "{% raw %}\n")
-      IO.append(out1, IO.readBytes(conf1))
-      IO.write(out1, "\n{% endraw %}", append = true)
-      IO.write(out2, "{% raw %}\n")
-      IO.append(out2, IO.readBytes(conf2))
-      IO.write(out2, "\n{% endraw %}", append = true)
-      val oa1 = (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml"
-      val oaout = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.yml"
-      IO.copy(Seq(oa1 -> oaout))
-      Seq(out1, out2, oaout)
+      val jekyllOut = resourceManaged.value/"main"/"jekyll"
+      val logger = streams.value.log
+
+      val templates = Seq(
+        (resourceDirectory in (restserver, Compile)).value / "reference.conf" -> jekyllOut /"_includes"/"server.conf",
+        (resourceDirectory in (joex, Compile)).value / "reference.conf" -> jekyllOut/"_includes"/"joex.conf",
+        (LocalRootProject / baseDirectory).value / "tools" / "exim" / "exim.conf" -> jekyllOut/ "_includes"/"sample-exim.conf"
+      )
+      val res1 = templates.map { case (s, t) =>
+        logger.info(s"Copying $s -> $t")
+        IO.write(t, "{% raw %}\n")
+        IO.append(t, IO.readBytes(s))
+        IO.write(t, "\n{% endraw %}", append = true)
+        t
+      }
+
+      val files = Seq(
+        (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml" -> jekyllOut/"openapi"/"docspell-openapi.yml"
+      )
+      IO.copy(files)
+      res1 ++ files.map(_._2)
     }.taskValue,
     Compile/resourceGenerators += Def.task {
       val staticDoc = (restapi/Compile/openapiStaticDoc).value

modules/microsite/docs/doc/tools.md

@@ -16,3 +16,5 @@ for integrating docspell.
 - [Browser Extension](browserext) An extension for firefox to
   upload files from your browser via *right-click -> upload to
   docspell*.
+- [SMTP Gateway](smtpgateway) Start a SMTP server that forwards all
+  mails to docspell.

modules/microsite/docs/doc/tools/smtpgateway.md

@@ -0,0 +1,195 @@
+---
+layout: docs
+title: SMTP Gateway with Exim
+permalink: doc/tools/smtpgateway
+---
+
+# {{ page.title }}
+
+One possible use case for the [integration
+endpoint](../uploading#integration-endpoint) is a SMTP server that
+forwards all local mail to docspell. This way there is no periodic
+polling involved and documents (e-mails) get into docspell without
+delay.
+
+The `tools/exim` folder contains a docker file and a sample
+`exim.conf` to help start with this setup. Note that these files
+provide a minimal setup, you might want to add tls and spam protection
+when opening it to the public.
+
+
+## What you need
+
+You need to own a domain and add the appropriate MX records to point
+to your server. In this document, the domain `test.org` is used.
+
+You need to enable the [integration
+endpoint](../uploading#integration-endpoint) in the docspell
+configuration.
+
+## Exim
+
+[Exim](http://exim.org/) is a popular smtp server (message transfer
+agent). It is used here only because of previous knowledge, but same
+can be achieved with other MTAs.
+
+
+## The Config File
+
+Here is the example config file for exim:
+
+```
+{% include sample-exim.conf %}
+```
+
+Exim has good [documentation](https://www.exim.org/docs.html), look
+there for more info. The following is only a quick summary of the file
+above.
+
+The `domainlist local_domains` should list your domain. Only mails to
+this domain are allowed, as specified in the first rule in
+`acl_check_rcpt`. So mails to `name@test.org` are ok, but
+`name@someother.org` not.
+
+Another rule in `acl_check_rcpt` executes a `GET` request against the
+integration endpoint. If that fails, the recipient is wrong (or the
+endpoint disabled) and the mail is rejected right away.
+
+Then the `routers` define how a mail is handled. There is only one
+router that accepts all mails (that have not been rejected by a rule
+in acls) and uses the `docspell` transport to deliver it. The
+transport specifies a command via the `pipe` driver that is run with
+the mail. The mail itself is provided via stdin. So a simple `curl`
+command can upload it to the integration endpoint. Here are some quick
+notes about the used options (see `man curl`):
+
+- `--silent` and `--out /dev/null` don't print upload progress
+  information and no output to stdout
+- `--fail` return non-zero if http status code is not success
+- `-F` use a multipart/form-data request (defaults to a POST request)
+- `"file=@-;filename=\"$_subject:\""` add one part with name `file`
+  and take the data from stdin (`@-`). Since there is no filename, we
+  use the subject of the mail. This is [supported by
+  exim](http://exim.org/exim-html-current/doc/html/spec_html/ch-string_expansions.html)
+  by expanding the subject mail header via `$h_subject:` (the colon is
+  required).
+- `$local_part` this is expanded by exim to the recipient address,
+  only the part until the `@` sign.
+- `${env{DS_HEADER}{$value} fail}` looks up an environment variable by
+  key `DS_HEADER`. This is usually defined in `docker-compose.yml`.
+  The value must be the "secret" header value as defined in docspell's
+  configuration file.
+- `${env{DS_URL}{$value} fail}` the url to docspell. It is looked up
+  from the environment with key `DS_URL`, which is usually defined in
+  `docker-compose.yml`. Adding the `$local_part` at the end means that
+  mails to `somename@test.org` are uploaded to the collective
+  `somename`.
+
+
+## Install with Docker
+
+Go into the `tools/exim` directory and build the docker image:
+
+``` shell
+docker build -t ds-exim:latest -f exim.dockerfile .
+```
+
+Then start docspell somewhere and configure the integration endpoint
+to use http-header protection; i.e. set this in the config file:
+
+```
+docspell.server {
+  integration-endpoint {
+    enabled = true
+    http-header = {
+      enabled = true
+      header-value = "test123"
+    }
+  }
+}
+```
+
+Then edit the `docker-compose.yml` and change the environment
+variables as needed.
+
+Finally start the container:
+
+``` shell
+docker-compose up
+```
+
+
+## Test Run
+
+Now it is possible to send mails to this MTA which will be immediatly
+uploaded to docspell for the collective corresponding to the
+`$local_part` of the recipients address. Here is a quick telnet
+session (the collective is named `family`):
+
+```
+fish ~> telnet localhost 25
+Trying ::1...
+Connected to localhost.
+Escape character is '^]'.
+220 test.org ESMTP Exim 4.93 Sun, 14 Jun 2020 19:03:51 +0000
+ehlo localhost
+250-test.org Hello localhost [::1]
+250-SIZE 31457280
+250-8BITMIME
+250-PIPELINING
+250-CHUNKING
+250 HELP
+mail from:<me@test.org>
+250 OK
+rcpt to:<family@test.org>
+250 Accepted
+data
+354 Enter message, ending with "." on a line by itself
+From: me@test.org
+To: family@test.org
+Subject: This is a test
+
+Test,
+
+this is just a test mail.
+.
+250 OK id=1jkXwf-000007-0d
+quit
+221 test.org closing connection
+Connection closed by foreign host.
+fish ~>
+```
+
+The mail is processed and results in an item:
+
+<div class="thumbnail">
+  <img src="../../img/exim-mail.png">
+</div>
+
+However, if a mail is to an unknown collective or not to the
+configured local domain, the server rejects it immediately:
+
+``` shell
+fish ~> telnet localhost 25
+Trying ::1...
+Connected to localhost.
+Escape character is '^]'.
+220 test.org ESMTP Exim 4.93 Sun, 14 Jun 2020 19:07:04 +0000
+ehlo localhost
+250-test.org Hello localhost [::1]
+250-SIZE 31457280
+250-8BITMIME
+250-PIPELINING
+250-CHUNKING
+250 HELP
+mail from:<me@test.org>
+250 OK
+rcpt to:<family22@test.org>
+550 Recipient unknown
+rcpt to:<family@gmail.com>
+550 Administrative prohibition
+quit
+221 test.org closing connection
+Connection closed by foreign host.
+fish ~>
+```

modules/microsite/docs/doc/uploading.md

@@ -109,6 +109,11 @@ The endpoint is disabled by default, an admin must change the
 `docspell.restserver.integration-endpoint.enabled` flag to `true` in
 the [configuration file](configure#rest-server).
 
+If queried by a `GET` request, it returns whether it is enabled and
+the collective exists.
+
+See the [SMTP gateway](tools/smtpgateway) for an example to use this
+endpoint.
 
 ## The Request
 

modules/microsite/docs/features.md

@@ -47,6 +47,8 @@ permalink: features
   - [Simple CLI for uploading files](doc/tools/ds)
   - [Firefox plugin](doc/tools/browserext): right click on a link and
     send the file to docspell
+  - [SMTP Gateway](doc/tools/smtpgateway): Setup a SMTP server that
+    delivers mails directly to docspell.
 - License: GPLv3
 
 

modules/microsite/src/main/resources/microsite/data/menu.yml

@@ -70,6 +70,9 @@ options:
       - title: Browser Extension (Firefox)
         url: doc/tools/browserext
 
+      - title: SMTP Gateway
+        url: doc/tools/smtpgateway
+
   - title: Api
     url: api
 

tools/exim/README.md

@@ -0,0 +1,14 @@
+# SMTP Gateway via Docker
+
+This is an example setup for a SMTP server that forwards all incoming
+mails to docspell via `curl`.
+
+The docker image contains [exim](https://exim.org) and a sample config
+file that runs curl against a configurable docspell url. It uses the
+[integration
+endpoint](https://docspell.org/doc/uploading#integration-endpoint) and
+it expects it to be configured with "http-header" protection. It can
+be easily adopted to use a different protection method.
+
+Please see the [documentation
+page](https://docspell.org/doc/tools/smtpgateway) for a guide.

tools/exim/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3.7'
+services:
+  smtp:
+    image: ds-exim:latest
+    ports:
+      - "25:25"
+    environment:
+      # This is the configured header value for the integration-endpoint
+      - DS_HEADER=test123
+      # This is the URL to docspell
+      - DS_URL=http://192.168.1.95:7880
+    # Use host network for demo purposes
+    network_mode: host

tools/exim/exim.conf

@@ -0,0 +1,57 @@
+## Provide certificates to enable StartTLS
+# tls_certificate = /var/lib/acme/test.org/fullchain.pem
+# tls_privatekey = /var/lib/acme/test.org/key.pem
+tls_advertise_hosts =
+
+primary_hostname = test.org
+domainlist local_domains = test.org
+timeout_frozen_after = 1m
+acl_smtp_rcpt = acl_check_rcpt
+acl_smtp_data = acl_check_data
+never_users = root
+host_lookup = *
+daemon_smtp_ports = 25
+
+message_size_limit = 30m
+
+keep_environment = DS_HEADER : DS_URL
+
+begin acl
+acl_check_rcpt:
+require
+  domains = +local_domains
+require
+  message = Sender verification failed
+  verify = sender
+require
+  message = Receiver verification failed
+  verify = recipient
+require
+  message = Recipient unknown
+  condition = ${run{/usr/bin/curl --out /dev/null --silent --fail -H "Docspell-Integration: ${env{DS_HEADER}{$value} fail}" "${env{DS_URL}{$value} fail}/api/v1/open/integration/item/$local_part"}{yes}{no}}
+warn
+  message = Reverse lookup failed
+  !verify = reverse_host_lookup
+accept
+
+acl_check_data:
+deny
+  message = Sender verification failed
+  !verify = header_sender
+accept
+
+begin routers
+local_users:
+  driver = accept
+  transport = docspell
+
+begin transports
+docspell:
+  driver = pipe
+  command = /usr/bin/curl --out /dev/null --silent --fail -H "Docspell-Integration: ${env{DS_HEADER}{$value} fail}" -F "file=@-;filename=\"$h_subject:\"" "${env{DS_URL}{$value} fail}/api/v1/open/integration/item/$local_part"
+  return_fail_output
+  user = nobody
+  delivery_date_add
+  envelope_to_add
+  return_path_add
+  log_output

tools/exim/exim.dockerfile

@@ -0,0 +1,8 @@
+FROM alpine:latest
+
+RUN apk add --no-cache exim curl
+USER exim
+COPY ./exim.conf /etc/exim/
+EXPOSE 25
+ENTRYPOINT ["exim"]
+CMD ["-bdf", "-v", "-q1m"]