From b01cb7715e987076cc67d91f334e0449c2b26a1b Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Mon, 19 Jul 2021 01:39:00 +0200
Subject: [PATCH 1/6] Add documentation for the cli tool

---
 tools/webextension/native/native.py           |  10 +-
 website/site/content/docs/api/upload.md       |   9 +-
 website/site/content/docs/features/_index.md  |   1 +
 website/site/content/docs/install/docker.md   |  50 ++--
 .../site/content/docs/install/download_run.md |   6 +-
 website/site/content/docs/install/nix.md      |   6 +-
 .../site/content/docs/joex/file-processing.md |   4 +-
 website/site/content/docs/tools/_index.md     |   2 +-
 website/site/content/docs/tools/browserext.md |  20 +-
 website/site/content/docs/tools/cli.md        | 266 ++++++++++++++++++
 .../content/docs/tools/consumedir-cleaner.md  |  10 +-
 website/site/content/docs/tools/consumedir.md |  11 +-
 .../content/docs/tools/convert-all-pdf.md     |  12 +-
 website/site/content/docs/tools/ds.md         |  11 +-
 .../content/docs/tools/regenerate-previews.md |  10 +-
 .../site/content/docs/tools/reset-password.md |  12 +-
 website/site/content/docs/webapp/uploading.md |   4 +-
 .../site/templates/shortcodes/infobubble.html |   2 +-
 18 files changed, 383 insertions(+), 63 deletions(-)
 create mode 100644 website/site/content/docs/tools/cli.md

diff --git a/tools/webextension/native/native.py b/tools/webextension/native/native.py
index bb3f5300..b8bd0941 100755
--- a/tools/webextension/native/native.py
+++ b/tools/webextension/native/native.py
@@ -7,13 +7,13 @@ import os
 from os.path import expanduser
 import subprocess
 
-# The path to the ds.sh tool.
+# The path to the dsc tool.
 try:
     home = expanduser("~")
-    with open(home + '/.config/docspell/ds.cmd', 'r') as file:
-        DS_SH_CMD = file.read().replace('\n', '')
+    with open(home + '/.config/docspell/dsc.cmd', 'r') as file:
+        DSC_CMD = file.read().replace('\n', '')
 except:
-    DS_SH_CMD="ds.sh"
+    DSC_SH_CMD="dsc"
 
 
 # Read a message from stdin and decode it.
@@ -43,7 +43,7 @@ def send_message(encoded_message):
 while True:
     filename = get_message()
     FNULL = open(os.devnull, 'w')
-    rc = subprocess.call(args=[DS_SH_CMD, filename], stdout=FNULL, stderr=FNULL, close_fds=True)
+    rc = subprocess.call(args=[DSC_CMD, "upload", "--delete", filename], stdout=FNULL, stderr=FNULL, close_fds=True)
     os.remove(filename)
     if rc == 0:
         send_message(encode_message(rc))
diff --git a/website/site/content/docs/api/upload.md b/website/site/content/docs/api/upload.md
index 007b86a4..34eda0e9 100644
--- a/website/site/content/docs/api/upload.md
+++ b/website/site/content/docs/api/upload.md
@@ -94,8 +94,8 @@ specified via a JSON structure in a part with name `meta`:
 
 # Endpoints
 
-Docspell needs to the collective that owns the files. There are the
-following ways for this.
+Docspell needs to know the collective that owns the files. There are
+the following ways for this.
 
 
 ## Authenticated User
@@ -160,5 +160,6 @@ checksum with:
 /api/v1/open/integration/checkfile/[collective-name]/[sha256-checksum]
 ```
 
-See the [SMTP gateway](@/docs/tools/smtpgateway.md) or the [consumedir
-script](@/docs/tools/consumedir.md) for examples to use this endpoint.
+See the [SMTP gateway](@/docs/tools/smtpgateway.md) or the [dsc
+watch/upload](@/docs/tools/cli.md#docker) command for example can use
+this endpoint.
diff --git a/website/site/content/docs/features/_index.md b/website/site/content/docs/features/_index.md
index 5a3497d5..bf3eab03 100644
--- a/website/site/content/docs/features/_index.md
+++ b/website/site/content/docs/features/_index.md
@@ -87,3 +87,4 @@ considering docspell at the moment.
 
 - Documents cannot be modified.
 - You can remove and add documents but there is no versioning.
+- There are no user/groups nor permission management
diff --git a/website/site/content/docs/install/docker.md b/website/site/content/docs/install/docker.md
index 790929a1..4dcd6ee9 100644
--- a/website/site/content/docs/install/docker.md
+++ b/website/site/content/docs/install/docker.md
@@ -18,15 +18,16 @@ There are images for all components that are available from the github
 release page. The images contain all the necessary
 [prerequisites](@/docs/install/prereq.md).
 
-- `docspell-restserver` this images contains the http server
-- `docspell-joex` this image contains the job executor and all
+- `docspell/restserver` this images contains the http server
+- `docspell/joex` this image contains the job executor and all
   required software (ocrmypdf, unoconv etc) mentioned in
   [prerequisites](@/docs/install/prereq.md).
-- `docspell-tools` this image simply contains all the scripts from the
-  `tools/` folder. They are installed into a location in `$PATH`. It
-  doesn't specify a `CMD` or `ENTRYPOINT`, so you must choose which
-  script to run. The scripts are all prefixed by `ds-`. So to run the
-  `consumedir.sh` script, execute `ds-consumedir`.
+- `docspell/dsc` this is an image containing a
+  [cli](@/docs/tools/cli.md) for docspell that can be used to watch
+  directories for new files. It doesn't specify a `CMD` or
+  `ENTRYPOINT`, so you must specify the exact command to run. Here, it
+  is used to watch a directory for uploading files. This runs the `dsc
+  watch` command.
 
 ### Examples
 
@@ -90,20 +91,20 @@ After this `docker ps` should show these two containers. Go to
 When signing up, use the same name for collective and user and then
 login with this name.
 
-For the last part, we use the `docspell/tools` image to create another
+For the last part, we use the `docspell/dsc` image to create another
 container that watches a directory and pushes files to docspell.
 
 ``` bash
 $ docker run -d --name ds-consume \
     --network dsnet --ip 10.4.3.4 \
     -v /tmp/inbox:/var/inbox \
-    docspell/tools:latest ds-consumedir -imdv --iheader "Docspell-Integration:test123" \
-      --path /var/inbox "http://10.4.3.3:7880/api/v1/open/integration/item"
+    docspell/dsc:latest dsc -v -d http://10.4.3.3:7880 watch -r --delete -i \
+      --header "Docspell-Integration:test123" /var/inbox
 ```
 
-This starts the [consumedir](@/docs/tools/consumedir.md) script that
-watches a directory and uploads arriving files to docspell server.
-This requires the value from the `integration-endpoint` setting to be
+This starts the [dsc](@/docs/tools/cli.md) tool that watches a
+directory and uploads arriving files to the docspell server. This
+requires the value from the `integration-endpoint` setting to be
 allowed to upload files. It also requires you to explicitely enable
 this: go to *Collective Profile → Settings* and enable the
 *Integration Endpoint*. Then create a subdirectory in `/tmp/inbox`
@@ -111,11 +112,14 @@ with the name of the *collective* that you registered and place a file
 into the `/tmp/inbox/[collective]` directory. The file is pushed to
 docspell and processed shortly after.
 
-To see all available options, run the script with the `--help` option:
+To see all available options, run `dsc` with the `--help` option:
 ``` bash
-$ docker run docspell/tools:latest ds-consumedir --help
+$ docker run docspell/dsc:latest dsc --help
 ```
 
+Or just [download the
+binary](https://github.com/docspell/dsc/releases/latest), no docker
+required.
 
 Note that this is just an example and is only to demonstrate how to
 use the docker images. For instance, this setup does not provide
@@ -127,9 +131,9 @@ below.
 
 There is a [docker-compose](https://docs.docker.com/compose/) setup
 available in the `/docker/docker-compose` folder. This setup is
-similiar to the example above, adding fulltext search and a PostgreSQL
-database by using just one command. It's only a few steps to get
-started.
+similiar to the example above, but adding fulltext search and a
+PostgreSQL database by using just one command. It's only a few steps
+to get started.
 
 ### Start Docspell
 #### 1. Get the docker-compose files
@@ -172,10 +176,12 @@ $ export DOCSPELL_HEADER_VALUE="my-secret-123"
 $ docker-compose up
 ```
 
-The environment variable defines a secret that is shared between some
-containers. You can define whatever you like. Please see the
-[consumedir.sh](@/docs/tools/consumedir.md#docker) docs for additional
-info.
+The environment variable defines a secret that is shared between the
+container watching a directory and the server. It is the header
+defined for the [integration
+endpoint](@/docs/api/upload.md#integration-endpoint) containers. You
+can use whatever you like. Please see the help to the [dsc
+tool](@/docs/tools/cli.md) docs for additional info.
 
 Goto `http://localhost:7880`, signup and login. When signing up, you
 choose the same name for collective and user. Then login with this
diff --git a/website/site/content/docs/install/download_run.md b/website/site/content/docs/install/download_run.md
index 5ec930cc..03d82577 100644
--- a/website/site/content/docs/install/download_run.md
+++ b/website/site/content/docs/install/download_run.md
@@ -116,6 +116,6 @@ page](@/docs/configure/_index.md#full-text-search-solr).
 
 ### Watching a directory
 
-The [consumedir](@/docs/tools/consumedir.md) script can be used for
-this. Using systemd or something similar, it is possible to create a
-system service that runs the script in "watch mode".
+The [dsc](@/docs/tools/cli.md) tool with the `watch` subcommand can be
+used for this. Using systemd or something similar, it is possible to
+create a system service that runs the script in "watch mode".
diff --git a/website/site/content/docs/install/nix.md b/website/site/content/docs/install/nix.md
index 426cfee4..a60c24c3 100644
--- a/website/site/content/docs/install/nix.md
+++ b/website/site/content/docs/install/nix.md
@@ -84,8 +84,8 @@ There are the following modules provided:
 - joex
 - consumedir
 
-The `consumedir` module defines a systemd unit that starts the
-`consumedir.sh` script to watch one or more directories for new files.
+The `consumedir` module defines a systemd unit that starts the `dsc
+watch` command to watch one or more directories for new files.
 
 You need to import the `release.nix` file as described above in your
 `configuration.nix` and then append the docspell module to your list of
@@ -125,7 +125,7 @@ in
   services.docspell-consumedir = {
     enable = true;
     watchDirs = ["/tmp/test"];
-    urls = ["http://localhost:7880/api/v1/open/upload/item/the-source-id"];
+    source-id = "the-source-id";
   };
 
   ...
diff --git a/website/site/content/docs/joex/file-processing.md b/website/site/content/docs/joex/file-processing.md
index 92e9db7f..5ab0e0b1 100644
--- a/website/site/content/docs/joex/file-processing.md
+++ b/website/site/content/docs/joex/file-processing.md
@@ -323,8 +323,8 @@ docspell.joex {
 
 When this is changed, you must re-generate all preview images. Check
 the api for this, there is an endpoint to regenerate all preview
-images for a collective. There is also a bash script provided in the
-`tools/` directory that can be used to call this endpoint.
+images for a collective. The [cli tool](../../tools/cli/) can be
+used for this.
 
 {% end %}
 
diff --git a/website/site/content/docs/tools/_index.md b/website/site/content/docs/tools/_index.md
index 4627f68a..d4793cda 100644
--- a/website/site/content/docs/tools/_index.md
+++ b/website/site/content/docs/tools/_index.md
@@ -4,7 +4,7 @@ description = "There are several tools distributed with docspell, like a program
 weight = 60
 insert_anchor_links = "right"
 template = "pages.html"
-redirect_to = "docs/tools/ds"
+redirect_to = "docs/tools/cli/"
 sort_by = "weight"
 [extra]
 mktoc = false
diff --git a/website/site/content/docs/tools/browserext.md b/website/site/content/docs/tools/browserext.md
index 7560df2b..058c06b8 100644
--- a/website/site/content/docs/tools/browserext.md
+++ b/website/site/content/docs/tools/browserext.md
@@ -13,10 +13,10 @@ firefox.
 Installation is a bit complicated, since you need to install external
 tools and the web extension. Both work together.
 
-# Install `ds.sh`
+# Install `dsc`
 
-First copy the `ds.sh` tool somewhere in your `PATH`, maybe
-`/usr/local/bin` as described above.
+First copy the [dsc](@/docs/tools/cli.md) tool somewhere in your
+`PATH`, maybe `/usr/local/bin`.
 
 
 # Install the native part
@@ -44,10 +44,14 @@ for details.
 And you might want to modify this json file, so the path to the
 `native.py` script is correct (it must be absolute).
 
-If the `ds.sh` script is in your `$PATH`, then this should
-work. Otherwise, edit the `native.py` script and change the path to
-the tool. Or create a file `$HOME/.config/docspell/ds.cmd` whose
-content is the path to the `ds.sh` script.
+If the `dsc` tool is in your `$PATH`, then this should work. You need
+to provide a default source id in your `~/.config/dsc/config.toml` so
+that the upload command can be used without further arguments.
+
+Otherwise, edit the `native.py` script and change the path to the tool
+and/or the arguments. Or create a file
+`$HOME/.config/docspell/dsc.cmd` whose content is the path to the
+`dsc` tool.
 
 
 # Install the extension
@@ -73,7 +77,7 @@ alternatives:
 When you right click on a file link, there should be a context menu
 entry *'Docspell Upload Helper'*. The add-on will download this file
 using the browser and then send the file path to the `native.py`
-script. This script will in turn call `ds.sh` which finally uploads it
+script. This script will in turn call `dsc` which finally uploads it
 to your configured URLs.
 
 Open the Add-ons page (`Ctrl`+`Shift`+`A`), the new add-on should be
diff --git a/website/site/content/docs/tools/cli.md b/website/site/content/docs/tools/cli.md
new file mode 100644
index 00000000..d52b2bd5
--- /dev/null
+++ b/website/site/content/docs/tools/cli.md
@@ -0,0 +1,266 @@
++++
+title = "CLI"
+description = "A command line interface to."
+weight = 5
++++
+
+# Introduction
+
+The **d**oc**s**pell **c**lient, short
+[dsc](https://github.com/docspell/dsc), is a tool to use
+docspell through the command line. It is also aims to be useful for
+your own scripts and programs.
+
+It is supposed to replace most of the shell scripts from the `tools/`
+directory.
+
+It is a work in progress; eventually most of the
+[api](@/docs/api/_index.md) will be covered.
+
+# Usage
+
+Download the binary for your architecture from the [release
+page](https://github.com/docspell/dsc/releases/latest) and rename it
+to `dsc`. Then run `dsc help` to see an overview of all commands. The
+help of each command is available via `dsc help [command]` or `dsc
+[command] --help`.
+
+
+There are docker images at
+[dockerhub](https://hub.docker.com/repository/docker/docspell/dsc),
+but it's usually easier to just download the binary. They should work
+on most systems without additional setups.
+
+Below are some quick infos to get started, please see [the project
+page](https://github.com/docspell/dsc) for more info.
+
+
+## Configuration
+
+A configuration file can be used to have some predefined settings, for
+example the docspell url, the admin secret etc. They can be overriden
+by specifying them as options.
+
+The config looks like this:
+
+``` toml
+docspell_url = "http://localhost:7880"
+default_format = "Tabular"
+admin_secret = "admin123"
+default_account = "demo"
+pdf_viewer = ["zathura", "{}"]
+#pass_entry = "my/entry"
+```
+
+For linuxes, the default location is `~/.config/dsc/config.toml`. You
+can give a config file explicitly via an option or the environment
+variable `DSC_CONFIG`.
+
+If you use the [pass](https://passwordstore.org) password manager, you
+can add your password entry to the config file as well.
+
+## Output format
+
+The "output format" defines how the information is printed on screen.
+The default output format is `Tabular` which prints a simple table.
+This table can also be formatted as CSV using `csv` as output format.
+These two modes are intended for humans and they may not present all
+information available.
+
+Alternatively, there is `json` and `lisp` as output format. These are
+intended for machine consumption. They always contain all information.
+If you look for some detail, use for example `json` to get all data in
+a structured form. On the shell you can use the awesome tool
+[jq](https://stedolan.github.io/jq/) to get exactly what you want.
+
+## Login
+
+Many tasks require to be logged in. This can be done via the `login`
+subcommand. You can specify account and password or fallback to the
+values in the config file.
+
+Once logged in, the session token will be saved to the filesystem
+(next to the config file) and is used for subsequent commands. It is
+renewed if expiry is near. If you don't issue any commands for a while
+you need to `login` again.
+
+# Use Cases / Examples
+
+
+## Uploads files
+
+The `upload` subcommand can uploads files to docspell. This is the
+replacement for the `ds.sh` shell script.
+
+You can specify a list of files that are all being uploaded. This
+command doesn't require to be logged in, it can also upload via a
+[source id](@/docs/webapp/uploading.md#anonymous-upload) or via the
+[integration endpoint](@/docs/api/upload.md#integration-endpoint).
+
+A source id can be given in the config file, then there are no
+additional options required. The simplest form is this:
+
+``` shell
+❯ dsc upload *.pdf
+File already in Docspell: article-velo.pdf
+Adding to request: test-ocr.pdf
+Sending request …
+┌─────────┬──────────────────┐
+│ success │ message          │
+├─────────┼──────────────────┤
+│ true    │ Files submitted. │
+└─────────┴──────────────────┘
+```
+
+By default, duplicate files are detected and not uploaded. This
+uploads all files in one single request. By default, each file results
+in one item. Using `--single-item` all files can be put into one item.
+
+It is possible to specify certain metadata, like tags or a folder,
+that is then attached to the resulting item.
+
+
+## Upload by traversing a directory
+
+The above use case was about uploading files. Using the `upload`
+subcommand with the `--traverse` option, you can traverse directories
+and upload all files in them. In this mode, each file will be uploaded
+in a separate request, so the `--single-item` option cannot be used.
+
+There are options to exclude/include files based on a [glob
+pattern](https://docs.rs/glob/0.3.0/glob/struct.Pattern.html).
+
+``` shell
+❯ dsc upload --traverse .
+File already in Docspell: article-velo.pdf
+File already in Docspell: demo/dirc/scan.21-03-12.15-50-54.pdf
+File already in Docspell: demo/dirc/test-stamp.pdf
+File already in Docspell: demo/letter-de.pdf
+Uploading eike/keywords.pdf
+File already in Docspell: eike/large-file.pdf
+Uploading eike/letter-en.pdf
+File already in Docspell: test-ocr.pdf
+┌─────────┬────────────┐
+│ success │ message    │
+├─────────┼────────────┤
+│ true    │ Uploaded 2 │
+└─────────┴────────────┘
+```
+
+
+## Watch a directory
+
+The `watch` subcommand can be used to watch one or more directories
+and upload files when they arrive. It uses the `upload` command under
+the hood and therefore most options are also available here.
+
+It detects file creations and skips a rename within a watched folder.
+The flag `-r` or `--recursive` is required to recursively watch a
+directory.
+
+``` shell
+❯ dsc watch -r .
+Watching directory (Recursive): .
+Press Ctrl-C to quit.
+------------------------------------------------------------------------------
+Got: /home/eike/workspace/projects/dsc/local/files/./demo/letter-de.pdf
+Adding to request: /home/eike/workspace/projects/dsc/local/files/./demo/letter-de.pdf
+Sending request …
+Server: Files submitted.
+```
+
+The `--matches` option allows to define a pattern for files to include.
+
+If watching a directory is not possible due to system constraints
+(e.g. when using NFS or SAMBA shares), a less efficient option is to
+use the `upload` subcommand with `--poll` option which periodically
+traverses a directory.
+
+## Admin commands
+
+These are a set of commands that simply call a route at the server to
+submit a maintenance task or to reset the password of some user. These
+commands require the [admin
+secret](@/docs/configure/_index.md#admin-endpoint) either in the
+config file or as an argument.
+
+Reset user password:
+``` shell
+❯ dsc admin reset-password --account demo
+┌─────────┬──────────────┬──────────────────┐
+│ success │ new password │ message          │
+├─────────┼──────────────┼──────────────────┤
+│ true    │ 2q2UeCVvMYg  │ Password updated │
+└─────────┴──────────────┴──────────────────┘
+```
+
+Recreate fulltext index:
+``` shell
+❯ dsc admin --admin-secret admin123 recreate-index
+┌─────────┬─────────────────────────────────────┐
+│ success │ message                             │
+├─────────┼─────────────────────────────────────┤
+│ true    │ Full-text index will be re-created. │
+└─────────┴─────────────────────────────────────┘
+```
+
+## Search for items
+
+The `search` command takes a [query](@/docs/query/_index.md) and
+prints the results.
+
+``` shell
+❯ dsc search 'corr:*'
+┌──────────┬────────────────────────────┬───────────┬────────────┬─────┬───────────────────────────┬───────────────┬────────┬─────────────┬────────────┬───────┐
+│ id       │ name                       │ state     │ date       │ due │ correspondent             │ concerning    │ folder │ tags        │ fields     │ files │
+├──────────┼────────────────────────────┼───────────┼────────────┼─────┼───────────────────────────┼───────────────┼────────┼─────────────┼────────────┼───────┤
+│ HVK7JuCF │ test-ocr.pdf               │ created   │ 2021-07-18 │     │ Pancake Company           │               │        │ Certificate │            │ 1     │
+│ 3odNawKE │ letter-en.pdf              │ confirmed │ 2021-07-18 │     │ Pancake Company           │               │        │ invoice     │            │ 1     │
+│ 3MA5NdhS │ large-file.pdf             │ confirmed │ 2021-07-18 │     │ Axa                       │               │        │ Certificate │            │ 1     │
+│ HDumXkRm │ keywords.pdf               │ confirmed │ 2021-07-18 │     │ Pancake Company           │               │        │ invoice     │            │ 1     │
+│ 733gM656 │ test-stamp.pdf             │ created   │ 2021-07-18 │     │ Pancake Company           │               │        │ Contract    │            │ 1     │
+│ 8LiciitB │ scan.21-03-12.15-50-54.pdf │ confirmed │ 2021-07-18 │     │ Supermarket               │               │        │ Receipt     │ CHF 89.44  │ 1     │
+│ 8nFt2z7T │ article-velo.pdf           │ confirmed │ 2021-07-18 │     │ Supermarket/Rudolf Müller │ Rudolf Müller │        │ invoice     │ CHF 123.11 │ 1     │
+│ kfugGdXU │ letter-de.pdf              │ created   │ 2021-07-18 │     │ Axa                       │ Rudolf Müller │        │ invoice     │            │ 1     │
+└──────────┴────────────────────────────┴───────────┴────────────┴─────┴───────────────────────────┴───────────────┴────────┴─────────────┴────────────┴───────┘
+```
+
+The same can be formatted as json and, for example, only print the ids:
+``` shell
+❯ dsc -f json search 'corr:*' | jq '.groups[].items[].id'
+"HVK7JuCFt4W-qxkcwq1cWCV-dvpGo4DpZzU-Q16Xoujojas"
+"3odNawKE1Ek-YJrWfPzekAq-47cjt14sexd-GK35JAEAanx"
+"3MA5NdhSrbx-3JkjEpqHiyU-XyVNb15tioh-SUVjMLi1aoV"
+"HDumXkRmDea-dNryjtRjk3V-ysdJmJNQGQS-UFb4DWNZJ3F"
+"733gM656S4T-d4HmEgdAV6Z-9zuHAd3biKM-mBwNriZpqMB"
+"8LiciitBVTi-DTmgiEUdqAJ-xXPckMvFHMc-JSiJMYvLaWh"
+"8nFt2z7T9go-1qaCTTgodub-592n6gpmdNR-VRcyYAyT7qj"
+"kfugGdXUGUc-mReaUnJxyUL-R44Lf7yH6RK-2JbZ1bv7dw"
+```
+
+
+# Docker
+
+The provided docker-compose setup runs this script to watch a single
+directory, `./docs` in current directory, for new files. If a new file
+is detected, it is pushed to docspell.
+
+This utilizes the [integration
+endpoint](@/docs/api/upload.md#integration-endpoint), which is enabled
+in the config file, to allow uploading documents for all collectives.
+A subfolder must be created for each registered collective. The docker
+containers are configured to use http-header protection for the
+integration endpoint. This requires you to provide a secret, that is
+shared between the rest-server and the `dsc` tool. This can be done by
+defining an environment variable which gets picked up by the
+containers defined in `docker-compose.yml`:
+
+``` bash
+export DOCSPELL_HEADER_VALUE="my-secret"
+docker-compose up
+```
+
+
+Now you can create a folder `./docs/<collective-name>` and place all
+files in there that you want to import. Once dropped in this folder
+the `consumedir` container will push it to docspell.
diff --git a/website/site/content/docs/tools/consumedir-cleaner.md b/website/site/content/docs/tools/consumedir-cleaner.md
index fc1a756e..2c2e3f26 100644
--- a/website/site/content/docs/tools/consumedir-cleaner.md
+++ b/website/site/content/docs/tools/consumedir-cleaner.md
@@ -1,9 +1,15 @@
 +++
-title = "Directory Cleaner"
+title = "Directory Cleaner (⊗)"
 description = "Clean directories from files in docspell"
-weight = 32
+weight = 150
 +++
 
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `cleanup` or the `upload` command.
+{% end %}
+
 # Introduction
 
 This script is made for cleaning up the consumption directory used for
diff --git a/website/site/content/docs/tools/consumedir.md b/website/site/content/docs/tools/consumedir.md
index 2fc5a5f0..db82833d 100644
--- a/website/site/content/docs/tools/consumedir.md
+++ b/website/site/content/docs/tools/consumedir.md
@@ -1,9 +1,16 @@
 +++
-title = "Consume Directory"
+title = "Consume Directory (⊗)"
 description = "A script to watch a directory for new files and upload them to docspell."
-weight = 30
+weight = 110
 +++
 
+
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+You can use the `watch` command, or the `upload` command with `--poll`.
+{% end %}
+
 # Introduction
 
 The `consumerdir.sh` is a bash script that works in two modes:
diff --git a/website/site/content/docs/tools/convert-all-pdf.md b/website/site/content/docs/tools/convert-all-pdf.md
index 412c363b..9679b48f 100644
--- a/website/site/content/docs/tools/convert-all-pdf.md
+++ b/website/site/content/docs/tools/convert-all-pdf.md
@@ -1,9 +1,17 @@
 +++
-title = "Convert All PDFs"
+title = "Convert All PDFs (⊗)"
 description = "Convert all PDF files using OcrMyPdf."
-weight = 60
+weight = 160
 +++
 
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `convert-all-pdfs` admin command, e.g. `dsc admin
+convert-all-pdfs`.
+{% end %}
+
+
 # convert-all-pdf.sh
 
 With version 0.9.0 there was support added for another external tool,
diff --git a/website/site/content/docs/tools/ds.md b/website/site/content/docs/tools/ds.md
index e59bf842..4aa901cb 100644
--- a/website/site/content/docs/tools/ds.md
+++ b/website/site/content/docs/tools/ds.md
@@ -1,9 +1,16 @@
 +++
-title = "Upload CLI"
+title = "Upload CLI (⊗)"
 description = "A script to quickly upload files from the command line."
-weight = 10
+weight = 100
 +++
 
+
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `upload` command (or the `up` alias), like `dsc up *.pdf`.
+{% end %}
+
 # Introduction
 
 The `tools/ds.sh` is a bash script to quickly upload files from the
diff --git a/website/site/content/docs/tools/regenerate-previews.md b/website/site/content/docs/tools/regenerate-previews.md
index 114f1c40..a0fb2e65 100644
--- a/website/site/content/docs/tools/regenerate-previews.md
+++ b/website/site/content/docs/tools/regenerate-previews.md
@@ -1,9 +1,15 @@
 +++
-title = "Regenerate Preview Images"
+title = "Regenerate Preview Images (⊗)"
 description = "Re-generates all preview images."
-weight = 80
+weight = 130
 +++
 
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `generate-previews` admin command, e.g. `dsc admin generate-previews`.
+{% end %}
+
 # regenerate-previews.sh
 
 This is a simple bash script to trigger the endpoint that submits task
diff --git a/website/site/content/docs/tools/reset-password.md b/website/site/content/docs/tools/reset-password.md
index 9c34b667..84aa20ec 100644
--- a/website/site/content/docs/tools/reset-password.md
+++ b/website/site/content/docs/tools/reset-password.md
@@ -1,9 +1,17 @@
 +++
-title = "Reset Password"
+title = "Reset Password (⊗)"
 description = "Resets a user password."
-weight = 70
+weight = 120
 +++
 
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `reset-password` admin command, e.g. `dsc admin reset-password
+--account "smith/john"`, where `smith` is the collective id and `john`
+the username.
+{% end %}
+
 
 This script can be used to reset a user password. This can be done by
 admins, who know the `admin-endpoint.secret` value in the
diff --git a/website/site/content/docs/webapp/uploading.md b/website/site/content/docs/webapp/uploading.md
index 2047a1a8..015d931f 100644
--- a/website/site/content/docs/webapp/uploading.md
+++ b/website/site/content/docs/webapp/uploading.md
@@ -78,8 +78,8 @@ $ curl -XPOST -F file=@test.pdf http://192.168.1.95:7880/api/v1/open/upload/item
 {"success":true,"message":"Files submitted."}
 ```
 
-There is a [script provided](@/docs/tools/ds.md) that uses curl to
-upload files from the command line more conveniently.
+There is a [cli](@/docs/tools/cli.md) to upload files from the command
+line more conveniently.
 
 When files are uploaded to an source endpoint, the items resulting
 from this uploads are marked with the name of the source. So you know
diff --git a/website/site/templates/shortcodes/infobubble.html b/website/site/templates/shortcodes/infobubble.html
index b78b31d6..c2f27adf 100644
--- a/website/site/templates/shortcodes/infobubble.html
+++ b/website/site/templates/shortcodes/infobubble.html
@@ -1,4 +1,4 @@
-<div class="notification is-{{ mode }} is-light" style="z-index: -1">
+<div class="notification is-{{ mode }} is-light" style="z-index: 0">
     <div class="content ">
         <p class="title is-5">
             {{ title }}

From 96adde71721b6968ee1bf4b872d380ef693cc9cb Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Sun, 25 Jul 2021 00:14:34 +0200
Subject: [PATCH 2/6] Use the dsc tool for the consumedir docker container

---
 docker/docker-compose/docker-compose.yml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/docker/docker-compose/docker-compose.yml b/docker/docker-compose/docker-compose.yml
index 25a6139b..7e069111 100644
--- a/docker/docker-compose/docker-compose.yml
+++ b/docker/docker-compose/docker-compose.yml
@@ -30,16 +30,19 @@ services:
       - solr
 
   consumedir:
-    image: docspell/tools:latest
+    image: docspell/dsc:nightly
     container_name: docspell-consumedir
     command:
-      - ds-consumedir
-      - "-vmdi"
-      - "--path"
-      - "/opt/docs"
-      - "--iheader"
+      - dsc
+      - "-vv"
+      - "-d"
+      - "http://docspell-restserver:7880"
+      - "watch"
+      - "--delete"
+      - "-ir"
+      - "--header"
       - "Docspell-Integration:$DOCSPELL_HEADER_VALUE"
-      - "http://docspell-restserver:7880/api/v1/open/integration/item"
+      - "/opt/docs"
     restart: unless-stopped
     env_file: ./.env
     volumes:

From 2ae2af0752566eb600832ef4a75a18d5809d5b01 Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Mon, 26 Jul 2021 00:26:00 +0200
Subject: [PATCH 3/6] Fix feature list

---
 website/site/content/docs/features/_index.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/website/site/content/docs/features/_index.md b/website/site/content/docs/features/_index.md
index bf3eab03..14202090 100644
--- a/website/site/content/docs/features/_index.md
+++ b/website/site/content/docs/features/_index.md
@@ -64,19 +64,16 @@ description = "A list of features and limitations."
     - [eml](https://en.wikipedia.org/wiki/Email#Filename_extensions)
       (e-mail files in plain text MIME)
 - Tooling:
+  - [Command Line Interface](@/docs/tools/cli.md) allowing to upload
+    files, watch folders and many more!
   - [Android App](@/docs/tools/android.md) to quickly upload files
     from your android devices
-  - [Watch a folder](@/docs/tools/consumedir.md): watch folders for
-    changes and send files to docspell
-  - [Simple CLI for uploading files](@/docs/tools/ds.md)
   - [Firefox plugin](@/docs/tools/browserext.md): right click on a
     link and send the file to docspell
   - [SMTP Gateway](@/docs/tools/smtpgateway.md): Setup a SMTP server
     that delivers mails directly to docspell.
   - [Paperless Import](@/docs/tools/paperless-import.md) for importing
     your data from paperless
-  - [Directory Cleaner](@/docs/tools/consumedir-cleaner.md) clean
-    directories with files that have been uploaded to docspell
 - License: GPLv3
 
 

From 29b8b5e9c7aa559bcd5c512a707d4bb9660e8f6f Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Tue, 27 Jul 2021 00:12:14 +0200
Subject: [PATCH 4/6] Add docs about export command and deprecate export shell
 script

---
 website/site/content/docs/tools/cli.md        | 112 +++++++++++++++++-
 .../site/content/docs/tools/export-files.md   |  11 +-
 .../content/docs/tools/paperless-import.md    |   2 +-
 3 files changed, 121 insertions(+), 4 deletions(-)

diff --git a/website/site/content/docs/tools/cli.md b/website/site/content/docs/tools/cli.md
index d52b2bd5..7215738d 100644
--- a/website/site/content/docs/tools/cli.md
+++ b/website/site/content/docs/tools/cli.md
@@ -86,10 +86,13 @@ you need to `login` again.
 
 # Use Cases / Examples
 
+These are some examples. Each command has a good help explaining all
+the options. Run `dsc [subcommand] --help` to see it.
+
 
 ## Uploads files
 
-The `upload` subcommand can uploads files to docspell. This is the
+The `upload` subcommand can upload files to docspell. This is the
 replacement for the `ds.sh` shell script.
 
 You can specify a list of files that are all being uploaded. This
@@ -147,6 +150,9 @@ File already in Docspell: test-ocr.pdf
 └─────────┴────────────┘
 ```
 
+The `--poll` option allows to periodically traverse and upload
+directories.
+
 
 ## Watch a directory
 
@@ -176,6 +182,110 @@ If watching a directory is not possible due to system constraints
 use the `upload` subcommand with `--poll` option which periodically
 traverses a directory.
 
+## Download files
+
+The `download` command allows to download files that match a given
+query. It is possible to download them all flat into some directory or
+directly into a zip file. For example, download all files that are
+tagged with `todo` into a zip file:
+
+``` shell
+❯ dsc download --zip 'tag:todo'
+Zipping 2 attachments into docspell-files.zip
+Downloading DOC-20191223-155707.jpg.pdf …
+Downloading DOC-20200803-174448.jpg.pdf …
+```
+
+It downloads the converted PDF files by default, which can be changed
+via some options.
+
+``` shell
+❯ dsc download --zip --original 'tag:todo'
+Zipping original files of 2 attachments into docspell-files.zip
+Downloading DOC-20191223-155707.jpg …
+Downloading DOC-20200803-174448.jpg …
+```
+
+
+## Export data
+
+The `export` command allows to download all items with metadata and
+their files. It downloads all items by default, but a query is also
+supported.
+
+In contrast to the `download` command, this is intended for getting
+everything out of docspell in some independent format. Files are
+downloaded (only original files) and the items metadata is also
+stored. So you don't loose the tags and correspondents that are
+carefully maintained with each item.
+
+It expects one directory where it will create a specfific directory
+structure as follows:
+
+```
+exports/
+├── by_date
+│   ├── 2019-07
+│   ├── 2020-08
+|       ├── BV2po65mAFU-…-bqUiwjz8f2W -> ../../items/BV/BV2po65mAFU-…-bqUiwjz8f2W
+|       └── FTUnhZ3AE1H-…-RQ9KhtRi486 -> ../../items/FT/FTUnhZ3AE1H-…-RQ9KhtRi486
+│   ├── …
+│   └── 2021-07
+├── by_tag
+│   ├── Contract
+│   ├── Important
+│   ├── Invoice
+|   │   ├── 455h3cQNdna-…-t6dF7NjAuDm -> ../../items/45/455h3cQNdna-…-t6dF7NjAuDm
+|   │   ├── 5yQ95tQ4khY-…-S9KrxcbRkZR -> ../../items/5y/5yQ95tQ4khY-…-S9KrxcbRkZR
+|   │   ├── 7xoiE4XdwgD-…-Eb2S3BCSd38 -> ../../items/7x/7xoiE4XdwgD-…-Eb2S3BCSd38
+|   │   └── 93npVoA73Cx-…-BnxYNsf4Qvi -> ../../items/93/93npVoA73Cx-…-BnxYNsf4Qvi
+│   ├── …
+│   └── Todo
+└── items
+    ├── 45
+    |   └── 455h3cQNdna-w8oTEw9wtpE-G7bCJbVpZPw-t6dF7NjAuDm
+    |       ├── files
+    |       │   └── DOC-20200803-174448.jpg
+    |       └── metadata.json
+    ├── …
+    └── Hb
+```
+
+All your items are stored below the `items` directory. It contains
+subdirectories that are created from the first two characters of the
+item id. Inside this folder, a folder with the complete item id is
+created and there all the data to the item is stored: the metadata in
+`metadata.json` and all files below `files/`.
+
+The options `--date-links` and `--tag-links` create the other two
+folders: `by_tag` and `by_date`. In there you'll find symlinks into
+the `items` folder that are organized by some metadata, namely tag and
+the item date.
+
+Example run:
+``` shell
+❯ dsc export --all --date-links --tag-links --target exports
+Exported item: test3.zip
+Exported item: README.md
+Exported item: LICENSE.txt
+Exported item: TestRMarkdown.pdf
+Exported item: DOC-20191223-155729.jpg
+Exported item: DOC-20191223-155707.jpg
+Exported item: DOC-20200808-154204.jpg
+Exported item: DOC-20200807-115654.jpg
+Exported item: DOC-20200803-174448.jpg
+Exported item: DOC-20200803-174448.jpg
+Exported item: DOC-20200803-174448.jpg
+Exported item: DOC-20200803-174448.jpg
+Exported item: DOC-20200804-132305.jpg
+Exported item: DOC-20191223-155707.jpg
+Exported item: keyweb.eml
+Exported 15 items.
+```
+
+With output format `json` or `lisp` each item is printed instead in
+full detail.
+
 ## Admin commands
 
 These are a set of commands that simply call a route at the server to
diff --git a/website/site/content/docs/tools/export-files.md b/website/site/content/docs/tools/export-files.md
index 0e4a4d27..bc9c0286 100644
--- a/website/site/content/docs/tools/export-files.md
+++ b/website/site/content/docs/tools/export-files.md
@@ -1,9 +1,16 @@
 +++
-title = "Export Files"
+title = "Export Files (⊗)"
 description = "Downloads all files from docspell."
-weight = 65
+weight = 165
 +++
 
+{% infobubble(mode="info", title="⚠ Please note") %}
+This script is now obsolete, you can use the [**CLI tool**](../cli/) instead.
+
+Use the `export`  command, e.g. `dsc export --all --target .`.
+{% end %}
+
+
 # export-files.sh
 
 This script can be used to download all files from docspell that have
diff --git a/website/site/content/docs/tools/paperless-import.md b/website/site/content/docs/tools/paperless-import.md
index c1f02d29..6439f7af 100644
--- a/website/site/content/docs/tools/paperless-import.md
+++ b/website/site/content/docs/tools/paperless-import.md
@@ -1,7 +1,7 @@
 +++
 title = "Paperless Import"
 description = "Import your data from paperless."
-weight = 35
+weight = 60
 +++
 
 # Introduction

From 85e4ab78da1f92f03cc485e437c4d2e8b298164e Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Wed, 28 Jul 2021 23:25:01 +0200
Subject: [PATCH 5/6] Update asciinema for dsc

---
 website/site/content/docs/feed/_index.md | 17 +++++++----------
 website/site/content/docs/tools/cli.md   |  9 +++++++++
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/website/site/content/docs/feed/_index.md b/website/site/content/docs/feed/_index.md
index f193539a..7cc847e9 100644
--- a/website/site/content/docs/feed/_index.md
+++ b/website/site/content/docs/feed/_index.md
@@ -124,21 +124,18 @@ read more.
 
 # Command-Line
 
-I like to use the command line, so there is a small script that
-accepts one or more files and uploads them to a configured Docspell
-url. It is possible to let the script delete all successfully uploaded
-files.
+I like to use the command line, and so there is a cli that can be used
+for some tasks, for example uploading files. Below is a quick demo, it
+supports many more options, see the link below for details.
 
-<div class="columns is-centered">
-  <div class="column is-four-fifths">
-    <script id="asciicast-1GbSkyIq03hyzfGfvbcJRtSn8" src="https://asciinema.org/a/1GbSkyIq03hyzfGfvbcJRtSn8.js" async></script>
+<div class="columns is-centered is-full-width">
+  <div class="column">
+    <script id="asciicast-427679" src="https://asciinema.org/a/427679.js" async></script>
   </div>
 </div>
 
-The script can also be used to only check if a file already exists in
-your account.
 
-{{ buttonright(classes="is-primary ", href="/docs/tools/ds", text="More") }}
+{{ buttonright(classes="is-primary ", href="/docs/tools/cli", text="More") }}
 
 
 # Browser Extension
diff --git a/website/site/content/docs/tools/cli.md b/website/site/content/docs/tools/cli.md
index 7215738d..8352d45f 100644
--- a/website/site/content/docs/tools/cli.md
+++ b/website/site/content/docs/tools/cli.md
@@ -84,6 +84,15 @@ Once logged in, the session token will be saved to the filesystem
 renewed if expiry is near. If you don't issue any commands for a while
 you need to `login` again.
 
+## Demo
+
+<div class="columns is-centered is-full-width">
+  <div class="column">
+    <script id="asciicast-427679" src="https://asciinema.org/a/427679.js" async></script>
+  </div>
+</div>
+
+
 # Use Cases / Examples
 
 These are some examples. Each command has a good help explaining all

From 149f4d48ed03f44bb1f6e89e9ae89970d8850e21 Mon Sep 17 00:00:00 2001
From: eikek <eike.kettner@posteo.de>
Date: Wed, 28 Jul 2021 23:52:46 +0200
Subject: [PATCH 6/6] Update changelog

---
 Changelog.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/Changelog.md b/Changelog.md
index 537e7052..6deec6d8 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,28 @@
 
 *Unreleased*
 
+- Introducing a new CLI tool (#345) that replaces all the shell
+  scripts from the `tools/` directory! https://github.com/docspell/dsc
+- UI changes:
+  - year separators are now more prominent (#950)
+  - fixes a bug in the item counter in detail view when an item is
+    deleted (#920)
+  - German translation improvements (#901)
+  - The number of selected files is shown in upload page (#896)
+- The created date of an item can now be used in queries (#925, #958)
+- Setting tags api has been improved (#955)
+- Task for converting pdfs is now behind the admin secret (#949)
+- Task for generating preview images is now behind the admin secret (#915)
+- respond with 404 when the source-id is not correct (#931)
+- Update of core libraries (#890)
+- Add Japanese to the list of document languages (#948)
+- Fix setting the folder from metadata when processing a file and
+  allow to specifiy it by name or id (#940)
+- Fixes docspell config file in docker-compose setup (#909)
+- Fixes selecting the next job in the job executor (#898)
+- Fixes a bug that prevents uploading more than one file at once
+  (#938)
+
 ### Rest API Changes
 
 - Removed `sec/item/convertallpdfs` endpoint in favor for new
@@ -13,6 +35,14 @@
   `admin/attachments/generatePreviews` endpoint which is now an admin
   task to generate previews for all files. The now removed enpoint did
   this only for one collective.
+- `/sec/item/{id}/tags`: Setting tags to an item (replacing existing
+  tags) has been changed to allow tags to be specified as names or ids
+- `/sec/item/{id}/tagsremove`: Added a route to remove tags for a
+  single item
+  
+### Configuration Changes
+
+None.
 
 
 ## v0.24.0