diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 3a0cf295..025ea396 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -42,7 +42,6 @@ jobs: modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip - tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 02b4b56e..77654ad2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,4 +42,3 @@ jobs: modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip - tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip diff --git a/build.sbt b/build.sbt index 4f99fcdf..aa74d056 100644 --- a/build.sbt +++ b/build.sbt @@ -1038,7 +1038,7 @@ addCommandAlias( addCommandAlias("make-zip", ";restserver/Universal/packageBin ;joex/Universal/packageBin") addCommandAlias("make-deb", ";restserver/Debian/packageBin ;joex/Debian/packageBin") addCommandAlias("make-tools", ";root/toolsPackage") -addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb ;make-tools") +addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb") addCommandAlias("ci", "make; lint; test") addCommandAlias( diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..61b66ced --- /dev/null +++ b/tools/README.md @@ -0,0 +1,5 @@ +# Tools + +This is a place for some arbitrary tools. These are not packaged +within a release and are also not tested regularily and may be out of +date. But they may serve as input for ideas or as a starting point. diff --git a/tools/start-local/README.md b/tools/start-local/README.md new file mode 100644 index 00000000..c332f8d4 --- /dev/null +++ b/tools/start-local/README.md @@ -0,0 +1,22 @@ +# Start docspell quickly locally + +This is a bash script to quickly run docspell on your local machine. +It requires [tmux](https://github.com/tmux/tmux) to be installed (and +some others: curl, unzip etc). + +Any prerequisites are to be taken care of by yourself. Fulltext search +is disabled, unless a environment variable `DOCSPELL_SOLR_URL` exists. + +A H2 database is used by default, unless a env variable `DOCSPELL_DB` +exists. + +It then creates a configuration file, downloads docspell and starts +restserver and joex instances as given: + +```bash +❯ # start one joex and one restserver, use version 0.32.0 +❯ ./start-local.sh 0.32.0 1 1 + +❯ # start two joex and one restserver, use a nightly version +❯ ./start-local.sh 0.33.0-SNAPSHOT 1 2 +``` diff --git a/tools/start-local/start-local.sh b/tools/start-local/start-local.sh new file mode 100755 index 00000000..be2ff816 --- /dev/null +++ b/tools/start-local/start-local.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +# Start some number of docspell nodes. The database can be given as +# env variable, if not a h2 database is used. SOLR is only enabled, if +# a SOLR_URL env variable is available. +# +# You must have tmux installed as this is used to host the processes. + +set -euo pipefail + +ds_version=${1:-0.32.0-SNAPSHOT} +rest_nodes=${2:-1} +joex_nodes=${3:-1} + +rest_start_port=7880 +joex_start_port=8800 + +tmux_session_name="docspell-cluster" + +run_root=${DOCSPELL_CLUSTER_ROOT:-/tmp/docspell-cluster} + +default_db_url="jdbc:h2://$run_root/db/docspell-cluster.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" +db_url="${DOCSPELL_DB:-$default_db_url}" +db_user="${DOCSPELL_DB_USER:-dev}" +db_pass="${DOCSPELL_DB_USER:-dev}" +solr_url="${DOCSPELL_SOLR_URL:-none}" + +prepare_solr_config() { + local enable= + if [ "$solr_url" = "none" ]; then + solr_url="http://localhost" + enable="false" + else + enable="true" + fi + echo "enabled = $enable" + echo "solr.url = \"$solr_url\"" +} + +prepare_rest_config() { + port="$1" + app_id="$2" + file="$3" + + cat >>"$file" <<-EOF +docspell.server { + app-id = "$app_id" + base-url = "http://localhost:$port" + bind.address = "0.0.0.0" + bind.port = $port + full-text-search = { + $(prepare_solr_config) + } + auth.server-secret = "hex:caffee" + backend { + mail-debug = false + jdbc { + url = "${db_url}" + user = "${db_user}" + password = "${db_pass}" + } + signup { + mode = open + new-invite-password = "test" + } + } + admin-endpoint { + secret = "123" + } + integration-endpoint { + enabled = true + http-header = { + enabled = true + header-value = "test123" + } + } +} +EOF +} + +prepare_joex_config() { + port="$1" + app_id="$2" + file="$3" + + cat >> "$file" <<-EOF +docspell.joex { + app-id = "$app_id" + base-url = "http://localhost:$port" + bind.address = "0.0.0.0" + bind.port = $port + full-text-search = { + $(prepare_solr_config) + } + mail-debug = false + extraction { + preview.dpi = 64 + } + text-analysis { + nlp { + mode = full + clear-interval = "30 seconds" + } + } + jdbc { + url = "${db_url}" + user = "${db_user}" + password = "${db_pass}" + } + scheduler { + pool-size = 1 + wakeup-period = "10 minutes" + retries = 3 + retry-delay = "10 seconds" + } + house-keeping { + schedule = "*-*-* 01:00:00" + cleanup-invites = { + older-than = "10 days" + } + } + #convert.ocrmypdf.command.program = "/some/path/bin/ocrmypdf" +} +EOF +} + +prepare_root() { + rm -rf "$run_root/work" + mkdir -p "$run_root"/{db,work,pkg} +} + +get_session() { + set +e + session=$(tmux list-sessions | grep "$tmux_session_name" | cut -d':' -f1 | head -n1) + echo $session + set -e + } + +assert_no_session() { + local session=$(get_session) + if [ -n "$session" ]; then + echo "A tmux session already exists. Please stop this first." + exit 1 + fi +} + +prepare_project() { + local project="$1" + local number="$2" + local app_id="$project$number" + local wdir="$run_root/work/$project-$number" + mkdir -p "$wdir" + case "$project" in + "restserver") + local port=$(($rest_start_port + $number)) + prepare_rest_config $port $app_id "$wdir/ds.conf" + ;; + "joex") + local port=$(($joex_start_port + $number)) + prepare_joex_config $port $app_id "$wdir/ds.conf" + ;; + *) + echo "Unknown project: $project" + exit 1 + esac +} + +download_zip() { + if [ -f "$run_root/pkg/joex.zip" ] && [ -f "$run_root/pkg/restserver.zip" ]; then + echo "Not downloading, files already exist" + else + echo "Downloading docspell..." + if [[ $ds_version == *SNAPSHOT ]]; then + curl -#Lo "$run_root/pkg/joex.zip" "https://github.com/eikek/docspell/releases/download/nightly/docspell-joex-${ds_version}.zip" + else + curl -#Lo "$run_root/pkg/joex.zip" "https://github.com/eikek/docspell/releases/download/v${ds_version}/docspell-joex-${ds_version}.zip" + fi + if [[ $ds_version == *SNAPSHOT ]]; then + curl -#Lo "$run_root/pkg/restserver.zip" "https://github.com/eikek/docspell/releases/download/nightly/docspell-restserver-${ds_version}.zip" + else + curl -#Lo "$run_root/pkg/restserver.zip" "https://github.com/eikek/docspell/releases/download/v${ds_version}/docspell-restserver-${ds_version}.zip" + fi + fi + + echo "Unzipping..." + rm -rf "$run_root/pkg"/docspell-joex-* "$run_root/pkg"/docspell-restserver-* + unzip -qq "$run_root/pkg/restserver.zip" -d "$run_root/pkg" + unzip -qq "$run_root/pkg/joex.zip" -d "$run_root/pkg" +} + +start_project() { + local project="$1" + local number="$2" + + local wdir="$run_root/work/$project-$number" + local cfgfile="$wdir/ds.conf" + local bindir=$(realpath "$run_root/pkg"/docspell-$project-*/bin) + local session=$(get_session) + local tempdir="$wdir/tmp" + mkdir -p "$tempdir" + + if [ -z "$session" ]; then + echo "Starting in new session $project-$number..." + tmux new -d -s "$tmux_session_name" "cd $wdir && $bindir/docspell-$project -Djava.io.tmpdir=$tempdir -- $cfgfile" + else + echo "Starting $project-$number..." + tmux split-window -t "$tmux_session_name" "cd $wdir && $bindir/docspell-$project -Djava.io.tmpdir=$tempdir -- $cfgfile" + fi + sleep 1 +} + +## === Main + +assert_no_session + +echo "Version: $ds_version" +echo "Restserver nodes: $rest_nodes" +echo "Joex nodes: $joex_nodes" +echo "tmux session: $tmux_session_name" +echo "Working directory root: $run_root" +echo "Database: $db_url" +echo "SOLR: ${solr_url}" +echo "Continue?" +read + +prepare_root +download_zip + +n=0 +max=$(($rest_nodes > $joex_nodes ? rest_nodes : joex_nodes)) +while [ $n -lt $max ] +do + if [ $n -lt $rest_nodes ]; then + prepare_project "restserver" $n + start_project "restserver" $n + fi + if [ $n -lt $joex_nodes ]; then + prepare_project "joex" $n + start_project "joex" $n + fi + + n=$(($n + 1)) +done diff --git a/tools/webextension/README.md b/tools/webextension/README.md index 8014509c..ca73274a 100644 --- a/tools/webextension/README.md +++ b/tools/webextension/README.md @@ -1,10 +1,92 @@ # Webextension for Docspell -Idea: Inside the browser click on a PDF and send it to docspell. It is -downloaded in the context of your current page. Then handed to an -application that finally pushes it to docspell. +The idea is to click on a file in firefox and send it to docspell. It +is downloaded in the context of your current page. Then handed to an +application that pushes it to docspell. There is a browser add-on +implementing this in `tools/webextension`. This add-on only works with +firefox. -Please see the -[microsite](https://docspell.org/doc/tools) for -instructions or navigate to the corresponding [markdown -file](../../modules/microsite/src/main/tut/doc/tools.md#webextension-for-docspell). +Note: the same can be achieved by running the `dsc watch` on some +directory and let the browser (and other programs) download files in +this directory. + +---- + +Installation is a bit complicated, since you need to install external +tools and the web extension. Both work together. + +# Build it + +In the source root, run `sbt make-tools`. After it completes, there is +a zip file in the `tools/target` folder. Inside this zip, you can find +the web extension. + +# Install `dsc` + +First copy the [dsc](@/docs/tools/cli.md) tool somewhere in your +`PATH`, maybe `/usr/local/bin`. + + +# Install the native part + +Then install the "native" part of the web extension: + +Copy or symlink the `native.py` script into some known location. For +example: + +``` bash +ln -s ~/docspell-checkout/tools/webextension/native/native.py /usr/local/share/docspell/native.py +``` + +Then copy the `app_manifest.json` to +`$HOME/.mozilla/native-messaging-hosts/docspell.json`. For example: + +``` bash +cp ~/docspell-checkout/tools/webextension/native/app_manifest.json ~/.mozilla/native-messaging-hosts/docspell.json +``` + +See +[here](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Native_manifests#manifest_location) +for details. + +And you might want to modify this json file, so the path to the +`native.py` script is correct (it must be absolute). + +If the `dsc` tool is in your `$PATH`, then this should work. You need +to provide a default source id in your `~/.config/dsc/config.toml` so +that the upload command can be used without further arguments. + +Otherwise, edit the `native.py` script and change the path to the tool +and/or the arguments. Or create a file +`$HOME/.config/docspell/dsc.cmd` whose content is the path to the +`dsc` tool. + + +# Install the extension + +An extension file can be build using the `make-xpi.sh` script. But +installing it in "standard" firefox won't work, because [Mozilla +requires extensions to be signed by +them](https://wiki.mozilla.org/Add-ons/Extension_Signing). This means +creating an account and going through some process…. So here are two +alternatives: + +1. Open firefox and type `about:debugging` in the addressbar. Then + click on *'Load Temporary Add-on...'* and select the + `manifest.json` file. The extension is now installed. The downside + is, that the extension will be removed once firefox is closed. +2. Use Firefox ESR, which allows to install Add-ons not signed by + Mozilla. But it has to be configured: Open firefox and type + `about:config` in the address bar. Search for key + `xpinstall.signatures.required` and set it to `false`. This is + described on the last paragraph on [this + page](https://support.mozilla.org/en-US/kb/add-on-signing-in-firefox). + +When you right click on a file link, there should be a context menu +entry *'Docspell Upload Helper'*. The add-on will download this file +using the browser and then send the file path to the `native.py` +script. This script will in turn call `dsc` which finally uploads it +to your configured URLs. + +Open the Add-ons page (`Ctrl`+`Shift`+`A`), the new add-on should be +there.