diff --git a/build.sbt b/build.sbt index c31fd161..7e97df57 100644 --- a/build.sbt +++ b/build.sbt @@ -432,21 +432,27 @@ val microsite = project.in(file("modules/microsite")). ) ), Compile/resourceGenerators += Def.task { - val conf1 = (resourceDirectory in (restserver, Compile)).value / "reference.conf" - val conf2 = (resourceDirectory in (joex, Compile)).value / "reference.conf" - val out1 = resourceManaged.value/"main"/"jekyll"/"_includes"/"server.conf" - val out2 = resourceManaged.value/"main"/"jekyll"/"_includes"/"joex.conf" - streams.value.log.info(s"Copying reference.conf: $conf1 -> $out1, $conf2 -> $out2") - IO.write(out1, "{% raw %}\n") - IO.append(out1, IO.readBytes(conf1)) - IO.write(out1, "\n{% endraw %}", append = true) - IO.write(out2, "{% raw %}\n") - IO.append(out2, IO.readBytes(conf2)) - IO.write(out2, "\n{% endraw %}", append = true) - val oa1 = (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml" - val oaout = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.yml" - IO.copy(Seq(oa1 -> oaout)) - Seq(out1, out2, oaout) + val jekyllOut = resourceManaged.value/"main"/"jekyll" + val logger = streams.value.log + + val templates = Seq( + (resourceDirectory in (restserver, Compile)).value / "reference.conf" -> jekyllOut /"_includes"/"server.conf", + (resourceDirectory in (joex, Compile)).value / "reference.conf" -> jekyllOut/"_includes"/"joex.conf", + (LocalRootProject / baseDirectory).value / "tools" / "exim" / "exim.conf" -> jekyllOut/ "_includes"/"sample-exim.conf" + ) + val res1 = templates.map { case (s, t) => + logger.info(s"Copying $s -> $t") + IO.write(t, "{% raw %}\n") + IO.append(t, IO.readBytes(s)) + IO.write(t, "\n{% endraw %}", append = true) + t + } + + val files = Seq( + (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml" -> jekyllOut/"openapi"/"docspell-openapi.yml" + ) + IO.copy(files) + res1 ++ files.map(_._2) }.taskValue, Compile/resourceGenerators += Def.task { val staticDoc = (restapi/Compile/openapiStaticDoc).value diff --git a/modules/microsite/docs/doc/tools.md b/modules/microsite/docs/doc/tools.md index 6a05262a..8595ab3c 100644 --- a/modules/microsite/docs/doc/tools.md +++ b/modules/microsite/docs/doc/tools.md @@ -16,3 +16,5 @@ for integrating docspell. - [Browser Extension](browserext) An extension for firefox to upload files from your browser via *right-click -> upload to docspell*. +- [SMTP Gateway](smtpgateway) Start a SMTP server that forwards all + mails to docspell. diff --git a/modules/microsite/docs/doc/tools/smtpgateway.md b/modules/microsite/docs/doc/tools/smtpgateway.md new file mode 100644 index 00000000..9a73fdcf --- /dev/null +++ b/modules/microsite/docs/doc/tools/smtpgateway.md @@ -0,0 +1,195 @@ +--- +layout: docs +title: SMTP Gateway with Exim +permalink: doc/tools/smtpgateway +--- + +# {{ page.title }} + +One possible use case for the [integration +endpoint](../uploading#integration-endpoint) is a SMTP server that +forwards all local mail to docspell. This way there is no periodic +polling involved and documents (e-mails) get into docspell without +delay. + +The `tools/exim` folder contains a docker file and a sample +`exim.conf` to help start with this setup. Note that these files +provide a minimal setup, you might want to add tls and spam protection +when opening it to the public. + + +## What you need + +You need to own a domain and add the appropriate MX records to point +to your server. In this document, the domain `test.org` is used. + +You need to enable the [integration +endpoint](../uploading#integration-endpoint) in the docspell +configuration. + +## Exim + +[Exim](http://exim.org/) is a popular smtp server (message transfer +agent). It is used here only because of previous knowledge, but same +can be achieved with other MTAs. + + +## The Config File + +Here is the example config file for exim: + +``` +{% include sample-exim.conf %} +``` + +Exim has good [documentation](https://www.exim.org/docs.html), look +there for more info. The following is only a quick summary of the file +above. + +The `domainlist local_domains` should list your domain. Only mails to +this domain are allowed, as specified in the first rule in +`acl_check_rcpt`. So mails to `name@test.org` are ok, but +`name@someother.org` not. + +Another rule in `acl_check_rcpt` executes a `GET` request against the +integration endpoint. If that fails, the recipient is wrong (or the +endpoint disabled) and the mail is rejected right away. + +Then the `routers` define how a mail is handled. There is only one +router that accepts all mails (that have not been rejected by a rule +in acls) and uses the `docspell` transport to deliver it. The +transport specifies a command via the `pipe` driver that is run with +the mail. The mail itself is provided via stdin. So a simple `curl` +command can upload it to the integration endpoint. Here are some quick +notes about the used options (see `man curl`): + +- `--silent` and `--out /dev/null` don't print upload progress + information and no output to stdout +- `--fail` return non-zero if http status code is not success +- `-F` use a multipart/form-data request (defaults to a POST request) +- `"file=@-;filename=\"$_subject:\""` add one part with name `file` + and take the data from stdin (`@-`). Since there is no filename, we + use the subject of the mail. This is [supported by + exim](http://exim.org/exim-html-current/doc/html/spec_html/ch-string_expansions.html) + by expanding the subject mail header via `$h_subject:` (the colon is + required). +- `$local_part` this is expanded by exim to the recipient address, + only the part until the `@` sign. +- `${env{DS_HEADER}{$value} fail}` looks up an environment variable by + key `DS_HEADER`. This is usually defined in `docker-compose.yml`. + The value must be the "secret" header value as defined in docspell's + configuration file. +- `${env{DS_URL}{$value} fail}` the url to docspell. It is looked up + from the environment with key `DS_URL`, which is usually defined in + `docker-compose.yml`. Adding the `$local_part` at the end means that + mails to `somename@test.org` are uploaded to the collective + `somename`. + + +## Install with Docker + +Go into the `tools/exim` directory and build the docker image: + +``` shell +docker build -t ds-exim:latest -f exim.dockerfile . +``` + +Then start docspell somewhere and configure the integration endpoint +to use http-header protection; i.e. set this in the config file: + +``` +docspell.server { + integration-endpoint { + enabled = true + http-header = { + enabled = true + header-value = "test123" + } + } +} +``` + +Then edit the `docker-compose.yml` and change the environment +variables as needed. + +Finally start the container: + +``` shell +docker-compose up +``` + + +## Test Run + +Now it is possible to send mails to this MTA which will be immediatly +uploaded to docspell for the collective corresponding to the +`$local_part` of the recipients address. Here is a quick telnet +session (the collective is named `family`): + +``` +fish ~> telnet localhost 25 +Trying ::1... +Connected to localhost. +Escape character is '^]'. +220 test.org ESMTP Exim 4.93 Sun, 14 Jun 2020 19:03:51 +0000 +ehlo localhost +250-test.org Hello localhost [::1] +250-SIZE 31457280 +250-8BITMIME +250-PIPELINING +250-CHUNKING +250 HELP +mail from: +250 OK +rcpt to: +250 Accepted +data +354 Enter message, ending with "." on a line by itself +From: me@test.org +To: family@test.org +Subject: This is a test + +Test, + +this is just a test mail. +. +250 OK id=1jkXwf-000007-0d +quit +221 test.org closing connection +Connection closed by foreign host. +fish ~> +``` + +The mail is processed and results in an item: + +
+ +
+ +However, if a mail is to an unknown collective or not to the +configured local domain, the server rejects it immediately: + +``` shell +fish ~> telnet localhost 25 +Trying ::1... +Connected to localhost. +Escape character is '^]'. +220 test.org ESMTP Exim 4.93 Sun, 14 Jun 2020 19:07:04 +0000 +ehlo localhost +250-test.org Hello localhost [::1] +250-SIZE 31457280 +250-8BITMIME +250-PIPELINING +250-CHUNKING +250 HELP +mail from: +250 OK +rcpt to: +550 Recipient unknown +rcpt to: +550 Administrative prohibition +quit +221 test.org closing connection +Connection closed by foreign host. +fish ~> +``` diff --git a/modules/microsite/docs/doc/uploading.md b/modules/microsite/docs/doc/uploading.md index 9b427dd7..b827246c 100644 --- a/modules/microsite/docs/doc/uploading.md +++ b/modules/microsite/docs/doc/uploading.md @@ -109,6 +109,11 @@ The endpoint is disabled by default, an admin must change the `docspell.restserver.integration-endpoint.enabled` flag to `true` in the [configuration file](configure#rest-server). +If queried by a `GET` request, it returns whether it is enabled and +the collective exists. + +See the [SMTP gateway](tools/smtpgateway) for an example to use this +endpoint. ## The Request diff --git a/modules/microsite/docs/features.md b/modules/microsite/docs/features.md index 1db016c3..61694b2c 100644 --- a/modules/microsite/docs/features.md +++ b/modules/microsite/docs/features.md @@ -47,6 +47,8 @@ permalink: features - [Simple CLI for uploading files](doc/tools/ds) - [Firefox plugin](doc/tools/browserext): right click on a link and send the file to docspell + - [SMTP Gateway](doc/tools/smtpgateway): Setup a SMTP server that + delivers mails directly to docspell. - License: GPLv3 diff --git a/modules/microsite/src/main/resources/microsite/data/menu.yml b/modules/microsite/src/main/resources/microsite/data/menu.yml index 94bd5c7c..8608ad37 100644 --- a/modules/microsite/src/main/resources/microsite/data/menu.yml +++ b/modules/microsite/src/main/resources/microsite/data/menu.yml @@ -70,6 +70,9 @@ options: - title: Browser Extension (Firefox) url: doc/tools/browserext + - title: SMTP Gateway + url: doc/tools/smtpgateway + - title: Api url: api diff --git a/modules/microsite/src/main/resources/microsite/img/exim-mail.png b/modules/microsite/src/main/resources/microsite/img/exim-mail.png new file mode 100644 index 00000000..aee357f1 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/exim-mail.png differ diff --git a/tools/exim/README.md b/tools/exim/README.md new file mode 100644 index 00000000..266a423c --- /dev/null +++ b/tools/exim/README.md @@ -0,0 +1,14 @@ +# SMTP Gateway via Docker + +This is an example setup for a SMTP server that forwards all incoming +mails to docspell via `curl`. + +The docker image contains [exim](https://exim.org) and a sample config +file that runs curl against a configurable docspell url. It uses the +[integration +endpoint](https://docspell.org/doc/uploading#integration-endpoint) and +it expects it to be configured with "http-header" protection. It can +be easily adopted to use a different protection method. + +Please see the [documentation +page](https://docspell.org/doc/tools/smtpgateway) for a guide. diff --git a/tools/exim/docker-compose.yml b/tools/exim/docker-compose.yml new file mode 100644 index 00000000..5060e2aa --- /dev/null +++ b/tools/exim/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.7' +services: + smtp: + image: ds-exim:latest + ports: + - "25:25" + environment: + # This is the configured header value for the integration-endpoint + - DS_HEADER=test123 + # This is the URL to docspell + - DS_URL=http://192.168.1.95:7880 + # Use host network for demo purposes + network_mode: host diff --git a/tools/exim/exim.conf b/tools/exim/exim.conf new file mode 100644 index 00000000..e2d7d6b4 --- /dev/null +++ b/tools/exim/exim.conf @@ -0,0 +1,57 @@ +## Provide certificates to enable StartTLS +# tls_certificate = /var/lib/acme/test.org/fullchain.pem +# tls_privatekey = /var/lib/acme/test.org/key.pem +tls_advertise_hosts = + +primary_hostname = test.org +domainlist local_domains = test.org +timeout_frozen_after = 1m +acl_smtp_rcpt = acl_check_rcpt +acl_smtp_data = acl_check_data +never_users = root +host_lookup = * +daemon_smtp_ports = 25 + +message_size_limit = 30m + +keep_environment = DS_HEADER : DS_URL + +begin acl +acl_check_rcpt: +require + domains = +local_domains +require + message = Sender verification failed + verify = sender +require + message = Receiver verification failed + verify = recipient +require + message = Recipient unknown + condition = ${run{/usr/bin/curl --out /dev/null --silent --fail -H "Docspell-Integration: ${env{DS_HEADER}{$value} fail}" "${env{DS_URL}{$value} fail}/api/v1/open/integration/item/$local_part"}{yes}{no}} +warn + message = Reverse lookup failed + !verify = reverse_host_lookup +accept + +acl_check_data: +deny + message = Sender verification failed + !verify = header_sender +accept + +begin routers +local_users: + driver = accept + transport = docspell + +begin transports +docspell: + driver = pipe + command = /usr/bin/curl --out /dev/null --silent --fail -H "Docspell-Integration: ${env{DS_HEADER}{$value} fail}" -F "file=@-;filename=\"$h_subject:\"" "${env{DS_URL}{$value} fail}/api/v1/open/integration/item/$local_part" + return_fail_output + user = nobody + delivery_date_add + envelope_to_add + return_path_add + log_output \ No newline at end of file diff --git a/tools/exim/exim.dockerfile b/tools/exim/exim.dockerfile new file mode 100644 index 00000000..9fd4fe03 --- /dev/null +++ b/tools/exim/exim.dockerfile @@ -0,0 +1,8 @@ +FROM alpine:latest + +RUN apk add --no-cache exim curl +USER exim +COPY ./exim.conf /etc/exim/ +EXPOSE 25 +ENTRYPOINT ["exim"] +CMD ["-bdf", "-v", "-q1m"]