Remove tools from a release

It doesn't contain anything generally useful anymore.

.github/workflows/release-nightly.yml

@@ -42,7 +42,6 @@ jobs:
             modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip
             modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb
             modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip
-            tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx

.github/workflows/release.yml

@@ -42,4 +42,3 @@ jobs:
             modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip
             modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb
             modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip
-            tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip

build.sbt

@@ -1038,7 +1038,7 @@ addCommandAlias(
 addCommandAlias("make-zip", ";restserver/Universal/packageBin ;joex/Universal/packageBin")
 addCommandAlias("make-deb", ";restserver/Debian/packageBin ;joex/Debian/packageBin")
 addCommandAlias("make-tools", ";root/toolsPackage")
-addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb ;make-tools")
+addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb")
 
 addCommandAlias("ci", "make; lint; test")
 addCommandAlias(

tools/README.md

@@ -0,0 +1,5 @@
+# Tools
+
+This is a place for some arbitrary tools. These are not packaged
+within a release and are also not tested regularily and may be out of
+date. But they may serve as input for ideas or as a starting point.

tools/start-local/README.md

@@ -0,0 +1,22 @@
+# Start docspell quickly locally
+
+This is a bash script to quickly run docspell on your local machine.
+It requires [tmux](https://github.com/tmux/tmux) to be installed (and
+some others: curl, unzip etc).
+
+Any prerequisites are to be taken care of by yourself. Fulltext search
+is disabled, unless a environment variable `DOCSPELL_SOLR_URL` exists.
+
+A H2 database is used by default, unless a env variable `DOCSPELL_DB`
+exists.
+
+It then creates a configuration file, downloads docspell and starts
+restserver and joex instances as given:
+
+```bash
+❯ # start one joex and one restserver, use version 0.32.0
+❯ ./start-local.sh 0.32.0 1 1
+
+❯ # start two joex and one restserver, use a nightly version
+❯ ./start-local.sh 0.33.0-SNAPSHOT 1 2
+```

tools/start-local/start-local.sh

@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+
+# Start some number of docspell nodes. The database can be given as
+# env variable, if not a h2 database is used. SOLR is only enabled, if
+# a SOLR_URL env variable is available.
+#
+# You must have tmux installed as this is used to host the processes.
+
+set -euo pipefail
+
+ds_version=${1:-0.32.0-SNAPSHOT}
+rest_nodes=${2:-1}
+joex_nodes=${3:-1}
+
+rest_start_port=7880
+joex_start_port=8800
+
+tmux_session_name="docspell-cluster"
+
+run_root=${DOCSPELL_CLUSTER_ROOT:-/tmp/docspell-cluster}
+
+default_db_url="jdbc:h2://$run_root/db/docspell-cluster.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE"
+db_url="${DOCSPELL_DB:-$default_db_url}"
+db_user="${DOCSPELL_DB_USER:-dev}"
+db_pass="${DOCSPELL_DB_USER:-dev}"
+solr_url="${DOCSPELL_SOLR_URL:-none}"
+
+prepare_solr_config() {
+    local enable=
+    if [ "$solr_url" = "none" ]; then
+        solr_url="http://localhost"
+        enable="false"
+    else
+        enable="true"
+    fi
+    echo "enabled = $enable"
+    echo "solr.url = \"$solr_url\""
+}
+
+prepare_rest_config() {
+    port="$1"
+    app_id="$2"
+    file="$3"
+
+    cat >>"$file" <<-EOF
+docspell.server {
+  app-id = "$app_id"
+  base-url = "http://localhost:$port"
+  bind.address = "0.0.0.0"
+  bind.port = $port
+  full-text-search = {
+    $(prepare_solr_config)
+  }
+  auth.server-secret = "hex:caffee"
+  backend {
+    mail-debug = false
+    jdbc {
+      url = "${db_url}"
+      user = "${db_user}"
+      password = "${db_pass}"
+    }
+    signup {
+      mode = open
+      new-invite-password = "test"
+    }
+  }
+  admin-endpoint {
+    secret = "123"
+  }
+  integration-endpoint {
+    enabled = true
+    http-header = {
+      enabled = true
+      header-value = "test123"
+    }
+  }
+}
+EOF
+}
+
+prepare_joex_config() {
+    port="$1"
+    app_id="$2"
+    file="$3"
+
+    cat >> "$file" <<-EOF
+docspell.joex {
+  app-id = "$app_id"
+  base-url = "http://localhost:$port"
+  bind.address = "0.0.0.0"
+  bind.port = $port
+  full-text-search = {
+   $(prepare_solr_config)
+  }
+  mail-debug = false
+  extraction {
+    preview.dpi = 64
+  }
+  text-analysis {
+    nlp {
+      mode = full
+      clear-interval = "30 seconds"
+    }
+  }
+  jdbc {
+    url = "${db_url}"
+    user = "${db_user}"
+    password = "${db_pass}"
+  }
+  scheduler {
+    pool-size = 1
+    wakeup-period = "10 minutes"
+    retries = 3
+    retry-delay = "10 seconds"
+  }
+  house-keeping {
+    schedule = "*-*-* 01:00:00"
+    cleanup-invites = {
+      older-than = "10 days"
+    }
+  }
+  #convert.ocrmypdf.command.program = "/some/path/bin/ocrmypdf"
+}
+EOF
+}
+
+prepare_root() {
+    rm -rf "$run_root/work"
+    mkdir -p "$run_root"/{db,work,pkg}
+}
+
+get_session() {
+    set +e
+    session=$(tmux list-sessions | grep "$tmux_session_name" | cut -d':' -f1 | head -n1)
+    echo $session
+    set -e
+ }
+
+assert_no_session() {
+    local session=$(get_session)
+    if [ -n "$session" ]; then
+        echo "A tmux session already exists. Please stop this first."
+        exit 1
+    fi
+}
+
+prepare_project() {
+    local project="$1"
+    local number="$2"
+    local app_id="$project$number"
+    local wdir="$run_root/work/$project-$number"
+    mkdir -p "$wdir"
+    case "$project" in
+        "restserver")
+            local port=$(($rest_start_port + $number))
+            prepare_rest_config $port $app_id "$wdir/ds.conf"
+            ;;
+        "joex")
+            local port=$(($joex_start_port + $number))
+            prepare_joex_config $port $app_id "$wdir/ds.conf"
+            ;;
+        *)
+            echo "Unknown project: $project"
+            exit 1
+    esac
+}
+
+download_zip() {
+    if [ -f "$run_root/pkg/joex.zip" ] && [ -f "$run_root/pkg/restserver.zip" ]; then
+        echo "Not downloading, files already exist"
+    else
+        echo "Downloading docspell..."
+        if [[ $ds_version == *SNAPSHOT ]]; then
+            curl -#Lo "$run_root/pkg/joex.zip" "https://github.com/eikek/docspell/releases/download/nightly/docspell-joex-${ds_version}.zip"
+        else
+            curl -#Lo "$run_root/pkg/joex.zip" "https://github.com/eikek/docspell/releases/download/v${ds_version}/docspell-joex-${ds_version}.zip"
+        fi
+        if [[ $ds_version == *SNAPSHOT ]]; then
+            curl -#Lo "$run_root/pkg/restserver.zip" "https://github.com/eikek/docspell/releases/download/nightly/docspell-restserver-${ds_version}.zip"
+        else
+            curl -#Lo "$run_root/pkg/restserver.zip" "https://github.com/eikek/docspell/releases/download/v${ds_version}/docspell-restserver-${ds_version}.zip"
+        fi
+    fi
+
+    echo "Unzipping..."
+    rm -rf "$run_root/pkg"/docspell-joex-* "$run_root/pkg"/docspell-restserver-*
+    unzip -qq "$run_root/pkg/restserver.zip" -d "$run_root/pkg"
+    unzip -qq "$run_root/pkg/joex.zip" -d "$run_root/pkg"
+}
+
+start_project() {
+    local project="$1"
+    local number="$2"
+
+    local wdir="$run_root/work/$project-$number"
+    local cfgfile="$wdir/ds.conf"
+    local bindir=$(realpath "$run_root/pkg"/docspell-$project-*/bin)
+    local session=$(get_session)
+    local tempdir="$wdir/tmp"
+    mkdir -p "$tempdir"
+
+    if [ -z "$session" ]; then
+        echo "Starting in new session $project-$number..."
+        tmux new -d -s "$tmux_session_name" "cd $wdir && $bindir/docspell-$project -Djava.io.tmpdir=$tempdir -- $cfgfile"
+    else
+        echo "Starting $project-$number..."
+        tmux split-window -t "$tmux_session_name" "cd $wdir && $bindir/docspell-$project -Djava.io.tmpdir=$tempdir -- $cfgfile"
+    fi
+    sleep 1
+}
+
+## === Main
+
+assert_no_session
+
+echo "Version: $ds_version"
+echo "Restserver nodes: $rest_nodes"
+echo "Joex nodes: $joex_nodes"
+echo "tmux session: $tmux_session_name"
+echo "Working directory root: $run_root"
+echo "Database: $db_url"
+echo "SOLR: ${solr_url}"
+echo "Continue?"
+read
+
+prepare_root
+download_zip
+
+n=0
+max=$(($rest_nodes > $joex_nodes ? rest_nodes : joex_nodes))
+while [ $n -lt $max ]
+do
+    if [ $n -lt $rest_nodes ]; then
+        prepare_project "restserver" $n
+        start_project "restserver" $n
+    fi
+    if [ $n -lt $joex_nodes ]; then
+        prepare_project "joex" $n
+        start_project "joex" $n
+    fi
+
+    n=$(($n + 1))
+done

tools/webextension/README.md

@@ -1,10 +1,92 @@
 # Webextension for Docspell
 
-Idea: Inside the browser click on a PDF and send it to docspell. It is
-downloaded in the context of your current page. Then handed to an
-application that finally pushes it to docspell.
+The idea is to click on a file in firefox and send it to docspell. It
+is downloaded in the context of your current page. Then handed to an
+application that pushes it to docspell. There is a browser add-on
+implementing this in `tools/webextension`. This add-on only works with
+firefox.
 
-Please see the
-[microsite](https://docspell.org/doc/tools) for
-instructions or navigate to the corresponding [markdown
-file](../../modules/microsite/src/main/tut/doc/tools.md#webextension-for-docspell).
+Note: the same can be achieved by running the `dsc watch` on some
+directory and let the browser (and other programs) download files in
+this directory.
+
+----
+
+Installation is a bit complicated, since you need to install external
+tools and the web extension. Both work together.
+
+# Build it
+
+In the source root, run `sbt make-tools`. After it completes, there is
+a zip file in the `tools/target` folder. Inside this zip, you can find
+the web extension.
+
+# Install `dsc`
+
+First copy the [dsc](@/docs/tools/cli.md) tool somewhere in your
+`PATH`, maybe `/usr/local/bin`.
+
+
+# Install the native part
+
+Then install the "native" part of the web extension:
+
+Copy or symlink the `native.py` script into some known location. For
+example:
+
+``` bash
+ln -s ~/docspell-checkout/tools/webextension/native/native.py /usr/local/share/docspell/native.py
+```
+
+Then copy the `app_manifest.json` to
+`$HOME/.mozilla/native-messaging-hosts/docspell.json`. For example:
+
+``` bash
+cp ~/docspell-checkout/tools/webextension/native/app_manifest.json  ~/.mozilla/native-messaging-hosts/docspell.json
+```
+
+See
+[here](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Native_manifests#manifest_location)
+for details.
+
+And you might want to modify this json file, so the path to the
+`native.py` script is correct (it must be absolute).
+
+If the `dsc` tool is in your `$PATH`, then this should work. You need
+to provide a default source id in your `~/.config/dsc/config.toml` so
+that the upload command can be used without further arguments.
+
+Otherwise, edit the `native.py` script and change the path to the tool
+and/or the arguments. Or create a file
+`$HOME/.config/docspell/dsc.cmd` whose content is the path to the
+`dsc` tool.
+
+
+# Install the extension
+
+An extension file can be build using the `make-xpi.sh` script. But
+installing it in "standard" firefox won't work, because [Mozilla
+requires extensions to be signed by
+them](https://wiki.mozilla.org/Add-ons/Extension_Signing). This means
+creating an account and going through some process…. So here are two
+alternatives:
+
+1. Open firefox and type `about:debugging` in the addressbar. Then
+   click on *'Load Temporary Add-on...'* and select the
+   `manifest.json` file. The extension is now installed. The downside
+   is, that the extension will be removed once firefox is closed.
+2. Use Firefox ESR, which allows to install Add-ons not signed by
+   Mozilla. But it has to be configured: Open firefox and type
+   `about:config` in the address bar. Search for key
+   `xpinstall.signatures.required` and set it to `false`. This is
+   described on the last paragraph on [this
+   page](https://support.mozilla.org/en-US/kb/add-on-signing-in-firefox).
+
+When you right click on a file link, there should be a context menu
+entry *'Docspell Upload Helper'*. The add-on will download this file
+using the browser and then send the file path to the `native.py`
+script. This script will in turn call `dsc` which finally uploads it
+to your configured URLs.
+
+Open the Add-ons page (`Ctrl`+`Shift`+`A`), the new add-on should be
+there.