Merge pull request #840 from eikek/feature/docker-rework

Feature/docker rework

.github/workflows/docker-image.yml

@@ -0,0 +1,22 @@
+name: Publish Docker Images
+on:
+  release:
+    types: [ published ]
+jobs:
+  docker-images:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 0
+      - name: Set current version
+        run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - name: Build Docker Images (${{ env.DOCSPELL_VERSION }})
+        run: ./docker/dockerfiles/build.sh ${{ env.DOCSPELL_VERSION }}
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Publish Images (${{ env.DOCSPELL_VERSION }})
+        run: ./docker/dockerfiles/push.sh ${{ env.DOCSPELL_VERSION }}

.github/workflows/release-nightly.yml

@@ -0,0 +1,52 @@
+name: Release Nightly
+on:
+  push:
+    branches:
+      - "master"
+jobs:
+  release-nightly:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        java: [ 'openjdk@1.11' ]
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 0
+      - uses: olafurpg/setup-scala@v10
+        with:
+          java-version: ${{ matrix.java }}
+      - uses: jorelali/setup-elm@v3
+        with:
+          elm-version: 0.19.1
+      - uses: bahmutov/npm-install@v1
+      - name: Coursier cache
+        uses: coursier/cache-action@v6
+      - name: Set current version
+        run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - name: sbt ci ${{ github.ref }}
+        run: sbt ci
+      - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
+        run: sbt make-pkg
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "nightly"
+          prerelease: true
+          title: "Docspell Nightly"
+          files: |
+            modules/restserver/target/docspell-restserver_${{ env.DOCSPELL_VERSION }}_all.deb
+            modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip
+            modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb
+            modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip
+            tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip
+      - name: Build Docker Images (${{ env.DOCSPELL_VERSION }})
+        run: ./docker/dockerfiles/build.sh ${{ env.DOCSPELL_VERSION }}
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Publish Images (${{ env.DOCSPELL_VERSION }})
+        run: ./docker/dockerfiles/push.sh ${{ env.DOCSPELL_VERSION }}

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: Release
+on:
+  push:
+    tags:
+      - 'v*'
+jobs:
+  release:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        java: [ 'openjdk@1.11' ]
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 0
+      - uses: olafurpg/setup-scala@v10
+        with:
+          java-version: ${{ matrix.java }}
+      - uses: jorelali/setup-elm@v3
+        with:
+          elm-version: 0.19.1
+      - uses: bahmutov/npm-install@v1
+      - name: Coursier cache
+        uses: coursier/cache-action@v6
+      - name: Set current version
+        run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
+        run: sbt make-pkg
+      - uses: meeDamian/github-release@2.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: Docspell ${{ env.DOCSPELL_VERSION }}
+          gzip: false
+          draft: true
+          prerelease: false
+          allow_override: false
+          files: >
+            modules/restserver/target/docspell-restserver_${{ env.DOCSPELL_VERSION }}_all.deb
+            modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip
+            modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb
+            modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip
+            tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip

.github/workflows/website.yml

@@ -0,0 +1,26 @@
+name: Publish Website
+on:
+  push:
+    branches:
+      - "current-docs"
+jobs:
+  publish-website:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          fetch-depth: 0
+      - name: Set current version
+        run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - uses: jorelali/setup-elm@v3
+        with:
+          elm-version: 0.19.1
+      - uses: cachix/install-nix-action@v13
+        with:
+          nix_path: nixpkgs=channel:nixos-20.09
+      - name: Print nixpkgs version
+        run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+      - name: Build website (${{ env.DOCSPELL_VERSION }})
+        run: nix-shell website/shell.nix --run "sbt make-website"
+      - name: Publish website (${{ env.DOCSPELL_VERSION }})
+        run: sbt publish-website

Changelog.md

@@ -895,10 +895,9 @@ improvements for processing files.
 - Adds the [full-text
   search](https://docspell.org/docs/webapp/finding/#full-text-search)
   feature (see #69). It requires a separate
-  [Solr](https://lucene.apache.org/solr) instance. Items can be
+  [Solr](https://solr.apache.org) instance. Items can be searched by
-  searched by documents contents and item/file names. It is possible
+  documents contents and item/file names. It is possible to use
-  to use full-text search to further confine the results via the
+  full-text search to further confine the results via the search menu.
-  search menu.
 - Fixes column types for item date and due-date for MariaDB (see #158)
   and adds an upper limit for due-dates (which is configurable).
 - Fixes a bug when cancelling jobs. Stuck jobs were only removed from

Contributing.md

@@ -63,5 +63,5 @@ nice functional language that compiles to javascript. The frontend is
 included in the http/restserver component. The CSS is provided by
 [TailwindCSS](https://tailwindcss.com/)..
 
-The [development](https://docspell.org/docs/dev/building/) page
+The [development](https://docspell.org/docs/dev) pages contains some
-contains some tips to get started.
+tips to get started.

build.sbt

@@ -776,8 +776,8 @@ addCommandAlias(
   "make",
   ";set webapp/elmCompileMode := ElmCompileMode.Production; set webapp/stylesMode := StylesMode.Prod ;root/openapiCodegen ;root/Test/compile"
 )
-addCommandAlias("make-zip", ";restserver/universal:packageBin ;joex/universal:packageBin")
+addCommandAlias("make-zip", ";restserver/Universal/packageBin ;joex/Universal/packageBin")
-addCommandAlias("make-deb", ";restserver/debian:packageBin ;joex/debian:packageBin")
+addCommandAlias("make-deb", ";restserver/Debian/packageBin ;joex/Debian/packageBin")
 addCommandAlias("make-tools", ";root/toolsPackage")
 addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb ;make-tools")
 
@@ -787,3 +787,5 @@ addCommandAlias(
   "scalafmtSbtCheck; scalafmtCheckAll; Compile/scalafix --check; Test/scalafix --check"
 )
 addCommandAlias("fix", "Compile/scalafix; Test/scalafix; scalafmtSbt; scalafmtAll")
+addCommandAlias("make-website", ";website/clean ;website/zolaBuild ;website/zolaCheck")
+addCommandAlias("publish-website", "website/publishToGitHubPages")

docker/docker-compose/.env

@@ -0,0 +1,2 @@
+TZ=Europe/Berlin
+DOCSPELL_HEADER_VALUE=none

docker/docker-compose/docker-compose.yml

@@ -0,0 +1,80 @@
+version: '3.8'
+services:
+
+  restserver:
+    image: docspell/restserver:latest
+    container_name: docspell-restserver
+    command:
+      - /opt/docspell.conf
+    restart: unless-stopped
+    ports:
+      - "7880:7880"
+    volumes:
+      - ./docspell.conf:/opt/docspell.conf
+    env_file: ./.env
+    depends_on:
+      - solr
+
+  joex:
+    image: docspell/joex:latest
+    container_name: docspell-joex
+    command:
+      - /opt/docspell.conf
+    restart: unless-stopped
+    env_file: ./.env
+    ports:
+      - "7878:7878"
+    volumes:
+      - ./docspell.conf:/opt/docspell.conf
+    depends_on:
+      - solr
+
+  consumedir:
+    image: docspell/tools:latest
+    container_name: docspell-consumedir
+    command:
+      - ds-consumedir
+      - "-vmdi"
+      - "--path"
+      - "/opt/docs"
+      - "--iheader"
+      - "Docspell-Integration:$DOCSPELL_HEADER_VALUE"
+      - "http://docspell-restserver:7880/api/v1/open/integration/item"
+    restart: unless-stopped
+    env_file: ./.env
+    volumes:
+      - ./docs:/opt/docs
+    depends_on:
+      - restserver
+
+  db:
+    image: postgres:13.3
+    container_name: postgres_db
+    restart: unless-stopped
+    volumes:
+      - docspell-postgres_data:/var/lib/postgresql/data/
+    environment:
+      - POSTGRES_USER=dbuser
+      - POSTGRES_PASSWORD=dbpass
+      - POSTGRES_DB=dbname
+
+  solr:
+    image: solr:8
+    container_name: docspell-solr
+    restart: unless-stopped
+    volumes:
+      - docspell-solr_data:/var/solr
+    command:
+      - solr-precreate
+      - docspell
+    healthcheck:
+      test: ["CMD", "curl", "f", "http://localhost:8983/solr/docspell/admin/ping"]
+      interval: 1m
+      timeout: 10s
+      retries: 2
+      start_period: 30s
+
+
+volumes:
+  docspell-postgres_data:
+  docspell-solr_data:

docker/docker-compose/docspell.conf

@@ -0,0 +1,83 @@
+# This is the configuration file for docspell. It contains two main
+# namespaces: docspell.server and docspell.joex. Each provide the
+# config for the respective component.
+#
+# They can be moved to different files, if necessary. For this example
+# though, both components are configured in this single file.
+#
+# Below are only some settings that differ from the default. Please
+# see https://docspell.org/docs/configure/#default-config for all
+# options and their documentation. This page provides more
+# information about the important config options.
+#
+# Note: this docker-compose setup is an example to get started. It
+# sets up one rest-server, one joex, a postgres database and a solr –
+# all on the same machine.
+
+# Define settings that are used in multiple places:
+db_url="jdbc:postgresql://db:5432/dbname"
+db_user="dbuser"
+db_pass="dbpass"
+solr_url="http://docspell-solr:8983/docspell"
+
+
+# This configures the restserver
+docspell.server {
+  base-url = "http://localhost:7880"
+  bind {
+    address = "0.0.0.0"
+  }
+  integration-endpoint {
+    enabled = true
+    http-header {
+      enabled = true
+      header-value = ${?DOCSPELL_HEADER_VALUE}
+    }
+  }
+  # Configuration of the full-text search engine.
+  full-text-search {
+    enabled = true
+    solr = {
+      url = ${solr_url}
+    }
+  }
+  backend {
+    jdbc {
+      url = ${db_url}
+      user = ${db_user}
+      password = ${db_pass}
+    }
+  }
+}
+
+
+# This configures joex
+#
+# Note to joex: It is currently setup for one instance. Should you
+# want to scale joex instance up (maybe to help processing a batch of
+# files), there are two options:
+#
+# - look at https://github.com/eikek/docspell/pull/552 to elastically
+#   start and stop joex instances via docker-compose
+# - set pool-size to some higher number; this requires to restart joex
+#
+docspell.joex {
+  base-url = "http://docspell-joex:7878"
+  bind {
+    address = "0.0.0.0"
+  }
+  jdbc {
+    url = ${db_url}
+    user = ${db_user}
+    password = ${db_pass}
+  }
+  full-text-search {
+    enabled = true
+    solr = {
+      url = ${solr_url}
+    }
+  }
+  scheduler {
+    pool-size = 1
+  }
+}

docker/dockerfiles/build.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+    echo "Please specify a version"
+    exit 1
+fi
+
+version="$1"
+if [[ $version == v* ]]; then
+    version="${version:1}"
+fi
+
+set -e
+cd "$(dirname "$0")"
+
+if [[ $version == *SNAPSHOT* ]]; then
+    echo ">>>> Building nightly images for $version <<<<<"
+    url_base="https://github.com/eikek/docspell/releases/download/nightly"
+
+    echo "============ Building Tools ============"
+    docker build \
+           --build-arg tools_url="$url_base/docspell-tools-$version.zip" \
+           --tag docspell/tools:nightly \
+           -f tools.dockerfile .
+
+    echo "============ Building Restserver ============"
+    docker build \
+           --build-arg restserver_url="$url_base/docspell-restserver-$version.zip" \
+           --tag docspell/restserver:nightly \
+           -f restserver.dockerfile .
+
+    echo "============ Building Joex ============"
+    docker build \
+           --build-arg joex_url="$url_base/docspell-joex-$version.zip" \
+           --tag docspell/joex:nightly \
+           -f joex.dockerfile .
+else
+    echo ">>>> Building release images for $version <<<<<"
+    echo "============ Building Tools ============"
+    docker build \
+           --build-arg version=$version \
+           --tag docspell/tools:v$version \
+           --tag docspell/tools:latest \
+           -f tools.dockerfile .
+
+    echo "============ Building Restserver ============"
+    docker build \
+           --build-arg version=$version \
+           --tag docspell/restserver:v$version \
+           --tag docspell/restserver:latest \
+           -f restserver.dockerfile .
+
+    echo "============ Building Joex ============"
+    docker build \
+           --build-arg version=$version \
+           --tag docspell/joex:v$version \
+           --tag docspell/joex:latest \
+           -f joex.dockerfile .
+fi

docker/dockerfiles/joex-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+echo "Starting unoconv listener"
+unoconv -l &
+
+/opt/docspell-joex/bin/docspell-joex "$@"

docker/dockerfiles/joex.dockerfile

@@ -0,0 +1,68 @@
+FROM alpine:latest
+
+ARG version=
+ARG joex_url=
+ARG UNO_URL=https://raw.githubusercontent.com/unoconv/unoconv/0.9.0/unoconv
+
+ENV JAVA_OPTS="-Xmx1536M"
+
+RUN apk add --no-cache openjdk11 \
+    tzdata \
+    bash \
+    curl \
+    ghostscript \
+    tesseract-ocr \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-fra \
+    tesseract-ocr-data-ita \
+    tesseract-ocr-data-spa \
+    tesseract-ocr-data-por \
+    tesseract-ocr-data-ces \
+    tesseract-ocr-data-nld \
+    tesseract-ocr-data-dan \
+    tesseract-ocr-data-fin \
+    tesseract-ocr-data-nor \
+    tesseract-ocr-data-swe \
+    tesseract-ocr-data-rus \
+    tesseract-ocr-data-ron \
+    tesseract-ocr-data-lav \
+    unpaper \
+    wkhtmltopdf \
+    libreoffice \
+    ttf-droid-nonlatin \
+    ttf-droid \
+    ttf-dejavu \
+    ttf-freefont \
+    ttf-liberation \
+    libxml2-dev \
+    libxslt-dev \
+    pngquant \
+    zlib-dev \
+    g++ \
+    qpdf \
+    py3-pip \
+    python3-dev \
+    libffi-dev\
+    qpdf-dev \
+    openssl-dev \
+    ocrmypdf \
+  && pip3 install --upgrade pip \
+  && pip3 install ocrmypdf \
+  && curl -Ls $UNO_URL -o /usr/local/bin/unoconv \
+  && chmod +x /usr/local/bin/unoconv \
+  && apk del curl libxml2-dev libxslt-dev zlib-dev g++ python3-dev py3-pip libffi-dev qpdf-dev openssl-dev \
+  && ln -s /usr/bin/python3 /usr/bin/python
+
+WORKDIR /opt
+RUN wget ${joex_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-joex-$version.zip} && \
+  unzip docspell-joex-*.zip && \
+  rm docspell-joex-*.zip && \
+  ln -snf docspell-joex-* docspell-joex
+
+COPY joex-entrypoint.sh /opt/joex-entrypoint.sh
+
+ENTRYPOINT ["/opt/joex-entrypoint.sh"]
+EXPOSE 7878
+
+HEALTHCHECK --interval=1m --timeout=10s --retries=2 --start-period=10s \
+  CMD pgrep -f joex/lib

docker/dockerfiles/push.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+
+if [ -z "$1" ]; then
+    echo "Please specify a version"
+    exit 1
+fi
+
+version="$1"
+if [[ $version == v* ]]; then
+    version="${version:1}"
+fi
+
+set -e
+cd "$(dirname "$0")"
+
+if [[ $version == *SNAPSHOT* ]]; then
+    echo "============ Push Tools ============"
+    docker push docspell/tools:nightly
+
+    echo "============ Push Restserver ============"
+    docker push docspell/restserver:nightly
+
+    echo "============ Push Joex ============"
+    docker push docspell/joex:nightly
+else
+    echo "============ Push Tools ============"
+    docker push docspell/tools:v$version
+    docker push docspell/tools:latest
+
+    echo "============ Push Restserver ============"
+    docker push docspell/restserver:v$version
+    docker push docspell/restserver:latest
+
+    echo "============ Push Joex ============"
+    docker push docspell/joex:v$version
+    docker push docspell/joex:latest
+fi

docker/dockerfiles/restserver.dockerfile

@@ -0,0 +1,18 @@
+FROM alpine:latest
+
+ARG version=
+ARG restserver_url=
+
+RUN apk add --no-cache openjdk11 bash tzdata
+
+WORKDIR /opt
+RUN wget ${restserver_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-restserver-$version.zip} && \
+  unzip docspell-restserver-*.zip && \
+  rm docspell-restserver-*.zip && \
+  ln -snf docspell-restserver-* docspell-restserver
+
+ENTRYPOINT ["/opt/docspell-restserver/bin/docspell-restserver"]
+EXPOSE 7880
+
+HEALTHCHECK --interval=1m --timeout=10s --retries=2 --start-period=30s \
+  CMD wget --spider http://localhost:7880

docker/dockerfiles/tools.dockerfile

@@ -0,0 +1,35 @@
+FROM alpine:latest
+
+# Builds an image where all scripts in tools/ are in PATH. There are
+# no assumptions what script to run, so there are no CMD or
+# ENTRYPOINTS defined.
+#
+# The scripts are named is in tools/ only prefixed by `ds-`
+#
+# Run the export-files script, for example:
+#
+#   docker run -e DS_USER=demo -e DS_PASS=test docspell/tools:dev ds-export-files "http://localhost" .
+#
+# The build requires to either specify a version build argument or a
+# tools_url build argument. If a tools_url argument is given, then
+# this url is used to download the tools zip file. Otherwise the
+# version argument is used to download from github.
+
+LABEL maintainer="eikek0 <eike@docspell.org>"
+
+ARG version=
+ARG tools_url=
+
+RUN apk add --no-cache curl bash inotify-tools jq sqlite
+
+WORKDIR /opt
+RUN wget ${tools_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-tools-$version.zip} && \
+  unzip docspell-tools-*.zip && \
+  rm docspell-tools-*.zip
+
+RUN bash -c 'while read f; do \
+    target="ds-$(basename "$f" ".sh")"; \
+    echo "Installing $f -> $target"; \
+    cp "$f" "/usr/local/bin/$target"; \
+    chmod 755 "/usr/local/bin/$target"; \
+  done < <(find /opt/docspell-tools-* -name "*.sh" -mindepth 2 -not -path "*webextension*")'

modules/joex/src/main/resources/reference.conf

@@ -575,7 +575,7 @@ docspell.joex {
       log-verbose = false
       # The defType parameter to lucene that defines the parser to
       # use. You might want to try "edismax" or look here:
-      # https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
+      # https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
       def-type = "lucene"
       # The default combiner for tokens. One of {AND, OR}.
       q-op = "OR"

modules/restserver/src/main/resources/reference.conf

@@ -145,7 +145,7 @@ docspell.server {
       log-verbose = false
       # The defType parameter to lucene that defines the parser to
       # use. You might want to try "edismax" or look here:
-      # https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
+      # https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
       def-type = "lucene"
       # The default combiner for tokens. One of {AND, OR}.
       q-op = "OR"

nix/module-joex.nix

@@ -1268,7 +1268,7 @@ in {
                     description = ''
                       The defType parameter to lucene that defines the parser to
                       use. You might want to try "edismax" or look here:
-                      https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
+                      https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
                     '';
                   };
                   q-op = mkOption {

nix/module-server.nix

@@ -397,7 +397,7 @@ in {
                     description = ''
                       The defType parameter to lucene that defines the parser to
                       use. You might want to try "edismax" or look here:
-                      https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
+                      https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
                     '';
                   };
                   q-op = mkOption {

tools/consumedir/consumedir.sh

@@ -22,7 +22,7 @@ if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
 fi
 
 OPTIONS=omhdp:vrmi
-LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:
+LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:,poll:
 
 ! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
 if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
@@ -36,7 +36,7 @@ eval set -- "$PARSED"
 
 declare -a watchdir
 help=n verbose=n delete=n once=n distinct=n recursive=n dryrun=n
-integration=n iuser="" iheader=""
+integration=n iuser="" iheader="" poll=""
 while true; do
     case "$1" in
         -h|--help)
@@ -84,6 +84,10 @@ while true; do
             iheader="$2"
             shift 2
             ;;
+        --poll)
+            poll="$2"
+            shift 2
+            ;;
         --)
             shift
             break
@@ -108,6 +112,8 @@ showUsage() {
     echo "  -h | --help           Prints this help text. (value: $help)"
     echo "  -m | --distinct       Optional. Upload only if the file doesn't already exist. (value: $distinct)"
     echo "  -o | --once           Instead of watching, upload all files in that dir. (value: $once)"
+    echo "       --poll <sec>     Run the script periodically instead of watching a directory. This can be"
+    echo "                        used if watching via inotify is not possible."
     echo "  -r | --recursive      Traverse the directory(ies) recursively (value: $recursive)"
     echo "  -i | --integration    Upload to the integration endpoint. It implies -r. This puts the script in"
     echo "                          a different mode, where the first subdirectory of any given starting point"
@@ -134,7 +140,7 @@ showUsage() {
     echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345"
     echo ""
     echo "Example: Integration Endpoint"
-    echo "$0 -i -iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item"
+    echo "$0 -i --iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item"
     echo ""
 }
 
@@ -355,11 +361,7 @@ checkSetup() {
     done
 }
 
-
+runOnce() {
-# warn if something seems not correctly configured
-checkSetup
-
-if [ "$once" = "y" ]; then
     info "Uploading all files (except hidden) in '$watchdir'."
     MD="-maxdepth 1"
     if [ "$recursive" = "y" ]; then
@@ -370,20 +372,37 @@ if [ "$once" = "y" ]; then
             process "$file" "$dir"
         done
     done
+}
+
+
+# warn if something seems not correctly configured
+checkSetup
+
+if [ "$once" = "y" ]; then
+    runOnce
 else
     REC=""
     if [ "$recursive" = "y" ]; then
         REC="-r"
     fi
-    $INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" |
+    if [ -z "$poll" ]; then
-        while read pathfile; do
+        $INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" |
-            if [[ "$(basename "$pathfile")" != .* ]]; then
+            while read pathfile; do
-                dir=$(findDir "$pathfile")
+                if [[ "$(basename "$pathfile")" != .* ]]; then
-                trace "The file '$pathfile' appeared below '$dir'"
+                    dir=$(findDir "$pathfile")
-                sleep 1
+                    trace "The file '$pathfile' appeared below '$dir'"
-                process "$(realpath "$pathfile")" "$dir"
+                    sleep 1
-            else
+                    process "$(realpath "$pathfile")" "$dir"
-                trace "Skip hidden file $(realpath "$pathfile")"
+                else
-            fi
+                    trace "Skip hidden file $(realpath "$pathfile")"
+                fi
+            done
+    else
+        echo "Running in polling mode: ${poll}s"
+        while [ : ]
+        do
+            runOnce
+            sleep $poll
         done
+    fi
 fi

website/elm/GetStarted.elm

@@ -20,7 +20,7 @@ getStarted version =
    Alternatively, [download](https://github.com/eikek/docspell/archive/master.zip) the sources and extract the zip file.
 2. Change into the `docker` directory:
    ```bash
-   $ cd docspell/docker
+   $ cd docspell/docker/docker-compose
    ```
 3. Run `docker-compose up`:
 
@@ -34,7 +34,7 @@ getStarted version =
    [`consumedir.sh`](doc/tools/consumedir#docker) docs for additional
    info.
 4. Goto <http://localhost:7880>, signup and login. When signing up,
-   you can choose the same name for collective and user. Then login
+   choose the same name for collective and user. Then login
    with this name and the password.
 
 5. (Optional) Create a folder `./docs/<collective-name>` (the name you

website/site/content/docs/configure/_index.md

@@ -7,12 +7,63 @@ weight = 40
 mktoc = true
 +++
 
-Docspell's executable can take one argument – a configuration file. If
+Docspell's executables (restserver and joex) can take one argument – a
-that is not given, the defaults are used. The config file overrides
+configuration file. If that is not given, the defaults are used. The
-default values, so only values that differ from the defaults are
+config file overrides default values, so only values that differ from
-necessary.
+the defaults are necessary. The complete default options and their
+documentation is at the end of this page.
+
+Besides the config file, another way is to provide individual settings
+via key-value pairs to the executable by the `-D` option. For example
+to override only `base-url` you could add the argument
+`-Ddocspell.server.base-url=…` to the command. Multiple options are
+possible. For more than few values this is very tedious, obviously, so
+the recommended way is to maintain a config file. If these options
+*and* a file is provded, then any setting given via the `-D…` option
+overrides the same setting from the config file.
+
+# File Format
+
+The format of the configuration files can be
+[HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation),
+JSON or what this [config
+library](https://github.com/lightbend/config) understands. The default
+values below are in HOCON format, which is recommended, since it
+allows comments and has some [advanced
+features](https://github.com/lightbend/config#features-of-hocon).
+Please refer to their documentation for more on this.
+
+A short description (please see the links for better understanding):
+The config consists of key-value pairs and can be written in a
+JSON-like format (called HOCON). Keys are organized in trees, and a
+key defines a full path into the tree. There are two ways:
+
+```
+a.b.c.d=15
+```
+
+or
+
+```
+a {
+  b {
+    c {
+      d = 15
+    }
+  }
+}
+```
+
+Both are exactly the same and these forms are both used at the same
+time. Usually the braces approach is used to group some more settings,
+for better readability.
+
+Strings that contain "not-so-common" characters should be enclosed in
+quotes. It is possible to define values at the top of the file and
+reuse them on different locations via the `${full.path.to.key}`
+syntax. When using these variables, they *must not* be enclosed in
+quotes.
 
-This applies to the restserver and the joex as well.
 
 # Important Config Options
 
@@ -95,7 +146,7 @@ endpoint.
 
 ## Full-Text Search: SOLR
 
-[Apache SOLR](https://lucene.apache.org/solr) is used to provide the
+[Apache SOLR](https://solr.apache.org) is used to provide the
 full-text search. Both docspell components must provide the same
 connection setup. This is defined in the `full-text-search.solr`
 subsection:
@@ -120,7 +171,7 @@ to `true`.
 
 When installing docspell manually, just install solr and create a core
 as described in the [solr
-documentation](https://lucene.apache.org/solr/guide/8_4/installing-solr.html).
+documentation](https://solr.apache.org/guide/8_4/installing-solr.html).
 That will provide you with the connection url (the last part is the
 core name).
 
@@ -143,7 +194,7 @@ by a joex instance eventually.
 Using this endpoint, the index will be re-created. This is sometimes
 necessary, for example if you upgrade SOLR or delete the core to
 provide a new one (see
-[here](https://lucene.apache.org/solr/guide/8_4/reindexing.html) for
+[here](https://solr.apache.org/guide/8_4/reindexing.html) for
 details). Note that a collective can also re-index their data using a
 similiar endpoint; but this is only deleting their data and doesn't do
 a full re-index.
@@ -394,53 +445,50 @@ The restserver component is very lightweight, here you can use
 defaults.
 
 
-# File Format
+# JVM Options
 
-The format of the configuration files can be
+The start scripts support some options to configure the JVM. One often
-[HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation),
+used setting is the maximum heap size of the JVM. By default, java
-JSON or whatever the used [config
+determines it based on properties of the current machine. You can
-library](https://github.com/lightbend/config) understands. The default
+specify it by given java startup options to the command:
-values below are in HOCON format, which is recommended, since it
-allows comments and has some [advanced
-features](https://github.com/lightbend/config#features-of-hocon).
-Please refer to their documentation for more on this.
-
-A short description (please see the links for better understanding):
-The config consists of key-value pairs and can be written in a
-JSON-like format (called HOCON). Keys are organized in trees, and a
-key defines a full path into the tree. There are two ways:
 
 ```
-a.b.c.d=15
+$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf
 ```
 
-or
+This would limit the maximum heap to 1GB. The double slash separates
+internal options and the arguments to the program. Another frequently
+used option is to change the default temp directory. Usually it is
+`/tmp`, but it may be desired to have a dedicated temp directory,
+which can be configured:
 
 ```
-a {
+$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf
-  b {
-    c {
-      d = 15
-    }
-  }
-}
 ```
 
-Both are exactly the same and these forms are both used at the same
+The command:
-time. Usually the braces approach is used to group some more settings,
-for better readability.
 
+```
+$ ./docspell-restserver*/bin/docspell-restserver -h
+```
 
-# Default Config
+gives an overview of supported options.
-## Rest Server
 
-{{ incl_conf(path="templates/shortcodes/server.conf") }}
+It is recommended to run joex with the G1GC enabled. If you use java8,
+you need to add an option to use G1GC (`-XX:+UseG1GC`), for java11
+this is not necessary (but doesn't hurt either). This could look like
+this:
 
+```
+./docspell-joex-{{version()}}/bin/docspell-joex -J-Xmx1596M -J-XX:+UseG1GC -- /path/to/joex.conf
+```
 
-## Joex
+Using these options you can define how much memory the JVM process is
+able to use. This might be necessary to adopt depending on the usage
+scenario and configured text analysis features.
 
+Please have a look at the corresponding [section](@/docs/configure/_index.md#memory-usage).
 
-{{ incl_conf(path="templates/shortcodes/joex.conf") }}
 
 
 # Logging
@@ -480,3 +528,15 @@ The `<root level="INFO">` means, that only log statements with level
 "INFO" will be printed. But the `<logger name="docspell"
 level="debug">` above says, that for loggers with name "docspell"
 statements with level "DEBUG" will be printed, too.
+
+
+# Default Config
+## Rest Server
+
+{{ incl_conf(path="templates/shortcodes/server.conf") }}
+
+
+## Joex
+
+
+{{ incl_conf(path="templates/shortcodes/joex.conf") }}

website/site/content/docs/dev/adr/0014_fulltext_search_engine.md

@@ -20,7 +20,7 @@ work (just the fulltext search is then not working).
 
 # Considered Options
 
-* [Apache SOLR](https://lucene.apache.org/solr)
+* [Apache SOLR](https://solr.apache.org)
 * [ElasticSearch](https://www.elastic.co/elasticsearch/)
 * [PostgreSQL](https://www.postgresql.org/docs/12/textsearch.html)
 * All of them or a subset

website/site/content/docs/dev/building.md

@@ -8,7 +8,8 @@ You must install [sbt](https://scala-sbt.org),
 [nodejs](https://www.npmjs.com/get-npm) (for the `npm` command) and
 [Elm](https://elm-lang.org).
 
-Clone the sources and run:
+Clone the sources, `cd` into the new directory and run `sbt`. This
+drops you in the sbt prompt. Then these tasks can be run:
 
 - `make` to compile all sources (Elm + Scala)
 - `make-zip` to create zip packages

website/site/content/docs/dev/development.md

@@ -139,9 +139,53 @@ sets more memory for the vm):
 
 ``` bash
 export QEMU_OPTS="-m 2048"
+export QEMU_NET_OPTS "hostfwd=tcp::7880-:7880"
 ./result/bin/run-docspelltest-vm
 ```
 
+# Release
+
+The CI and making a release is done via [github
+actions](https://docs.github.com/en/actions). The workflow is roughly
+like this:
+
+- each PR is only merged if the `sbt ci` task returns successfully.
+  This is ensured by the `ci.yml` workflow that triggers on each pull
+  request
+- each commit to the `master` branch is also going through `sbt ci`
+  and then a prerelease is created. The tag `nightly` is used to point
+  to the latest commit in `master`. Note, that this is [discouraged by
+  git](https://git-scm.com/docs/git-tag#_on_re_tagging), but github
+  doesn't allow to create a release without a tag. So this tag moves
+  (and is not really a tag then…). After the prerelease is created,
+  the docker images are built and pushed to docker hub into the
+  [docspell](https://hub.docker.com/u/docspell) organization. The
+  docker images are also tagged with `nightly` at docker hub. This is
+  all done via the `realease-nightly.yml` workflow.
+- A stable release is started by pushing a tag with pattern `v*` to
+  github. This triggers the `release.yml` workflow which builds the
+  packages and creates a release in *draft mode*. The `sbt ci` task
+  *is not* run, because it is meant to only release commits already in
+  the `master` branch. After this completes, the release notes need to
+  be added manually and then the release must be published at github.
+  This then triggers the `docker-images.yml` workflow, which builds
+  the corresponding docker images and pushes them to docker hub. The
+  docker images are tagged with the exact version and the `latest` tag
+  is moved to the new images. Another manual step is to set the branch
+  `current-docs` to its new state and push it to github. This will
+  trigger a build+publish of the website.
+- Publishing the website happens automatically on each push to the
+  branch `current-docs`. Changes to the current website must be based
+  on this branch.
+
+Some notes: I wanted a 2/3 step process when doing a stable release,
+to be able to add release notes manually (I don't want this to be
+automated right now) and to do some testing with the packages before
+publishing the release. However, for the nightly releases, this
+doesn't matter - everything must be automated here obviously. I also
+wanted the docker images to be built from the exact same artifacts
+that have been released at github (in contrast to being built again).
+
 
 # Background Info
 

website/site/content/docs/features/_index.md

@@ -13,7 +13,7 @@ description = "A list of features and limitations."
 - Handle multiple documents as one unit
 - OCR using [tesseract](https://github.com/tesseract-ocr/tesseract)
 - [Full-Text Search](@/docs/webapp/finding.md#full-text-search) based
-  on [Apache SOLR](https://lucene.apache.org/solr)
+  on [Apache SOLR](https://solr.apache.org)
 - Conversion to PDF: all files are converted into a PDF file. PDFs
   with only images (as often returned from scanners) are converted
   into searchable PDF/A pdfs.

website/site/content/docs/install/docker.md

@@ -0,0 +1,242 @@
++++
+title = "Docker"
+weight = 20
++++
+
+## Docker Images
+
+The docker images are at
+[hub.docker.com](https://hub.docker.com/u/docspell). The `latest` tag
+always points to the latest *release*. The releases are also tagged
+with their respective version number. Additionally, there are images
+tagged with `nightly` which are built from the `master` branch.
+Therefore the `nightly` packages should be used with care, because
+things might break in between. But they are useful for trying out
+something.
+
+There are images for all components that are available from the github
+release page. The images contain all the necessary
+[prerequisites](@/docs/install/prereq.md).
+
+- `docspell-restserver` this images contains the http server
+- `docspell-joex` this image contains the job executor and all
+  required software (ocrmypdf, unoconv etc) mentioned in
+  [prerequisites](@/docs/install/prereq.md).
+- `docspell-tools` this image simply contains all the scripts from the
+  `tools/` folder. They are installed into a location in `$PATH`. It
+  doesn't specify a `CMD` or `ENTRYPOINT`, so you must choose which
+  script to run. The scripts are all prefixed by `ds-`. So to run the
+  `consumedir.sh` script, execute `ds-consumedir`.
+
+### Examples
+
+These examples use `docker run` to start the restserver and
+jobexecutor. Both must be connected to the same database. For this
+example, a shared directory is used and the in-process database H2.
+For a real setup, using PostgreSQL is recommended.
+
+This requires to change the default config. This example creates a new
+config file. Please refer to the [configuration
+page](@/docs/configure/_index.md) for more details.
+
+``` bash
+$ cat > /tmp/docspell.conf <<-"EOF"
+# common settings
+db_url = "jdbc:h2:///var/docspell/db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE"
+
+# job executor settings
+docspell.joex.jdbc.url = ${db_url}
+docspell.joex.base-url = "http://10.4.3.2:7878"
+docspell.joex.bind.address = "0.0.0.0"
+
+# restserver settings
+docspell.server.backend.jdbc.url = ${db_url}
+docspell.server.bind.address = "0.0.0.0"
+docspell.server.integration-endpoint {
+  enabled = true
+  http-header {
+    enabled = true
+    header-value = "test123"
+  }
+}
+EOF
+```
+
+This sets the db url to the same value for both components; thus we
+can use the same file for both components. It also sets the bind
+address to bind the server socket on all interfaces. Another thing to
+note is the `base-url` setting for joex. This publishes joex by this
+ip, such that the server component can notify the job executor for new
+work. The `integration-endpoint` setting is explained later.
+
+After creating a common network, we start the rest server and joex:
+```
+$ docker network create --subnet 10.4.3.0/24 dsnet
+$ docker run -d --name ds-restserver \
+    --network dsnet --ip 10.4.3.3 \
+    -p 127.0.0.1:7880:7880 \
+    -v /tmp/testdb:/var/docspell \
+    -v /tmp/docspell.conf:/opt/docspell.conf \
+    docspell/restserver:latest /opt/docspell.conf
+$ docker run -d --name ds-joex \
+    --network dsnet --ip 10.4.3.2 \
+    -v /tmp/testdb:/var/docspell \
+    -v /tmp/docspell.conf:/opt/docspell.conf \
+    docspell/joex:latest /opt/docspell.conf
+```
+
+After this `docker ps` should show these two containers. Go to
+`http://localhost:7880` and sign up/login and start playing around.
+When signing up, use the same name for collective and user and then
+login with this name.
+
+For the last part, we use the `docspell/tools` image to create another
+container that watches a directory and pushes files to docspell.
+
+``` bash
+$ docker run -d --name ds-consume \
+    --network dsnet --ip 10.4.3.4 \
+    -v /tmp/inbox:/var/inbox \
+    docspell/tools:latest ds-consumedir -imdv --iheader "Docspell-Integration:test123" \
+      --path /var/inbox "http://10.4.3.3:7880/api/v1/open/integration/item"
+```
+
+This starts the [consumedir](@/docs/tools/consumedir.md) script that
+watches a directory and uploads arriving files to docspell server.
+This requires the value from the `integration-endpoint` setting to be
+allowed to upload files. It also requires you to explicitely enable
+this: go to *Collective Profile → Settings* and enable the
+*Integration Endpoint*. Then create a subdirectory in `/tmp/inbox`
+with the name of the *collective* that you registered and place a file
+into the `/tmp/inbox/[collective]` directory. The file is pushed to
+docspell and processed shortly after.
+
+To see all available options, run the script with the `--help` option:
+``` bash
+$ docker run docspell/tools:latest ds-consumedir --help
+```
+
+
+Note that this is just an example and is only to demonstrate how to
+use the docker images. For instance, this setup does not provide
+fulltext search. For a more sophisticated docker setup, use
+appropriate tools, for example `docker-compose` which is explained
+below.
+
+## Docker Compose
+
+There is a [docker-compose](https://docs.docker.com/compose/) setup
+available in the `/docker/docker-compose` folder. This setup is
+similiar to the example above, adding fulltext search and a PostgreSQL
+database by using just one command. It's only 3 steps to get started:
+
+### Start Docspell
+#### Get the docker-compose files
+
+Either via cloning the whole repository:
+
+```bash
+$ git clone https://github.com/eikek/docspell
+```
+
+This downloads all sources. What you need is only one subdirectory. So
+if you don't have git or don't want to clone the whole repo, use these
+steps instead:
+
+``` bash
+$ mkdir -p docspell/docker/docker-compose
+$ cd docspell/docker/docker-compose
+$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/docker-compose.yml
+$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/docspell.conf
+$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/.env
+```
+
+You can choose any directory instead of
+`docspell/docker/docker-compose`, of course.
+
+Change into the new `docker-compose` directory, for example:
+
+```bash
+$ cd docspell/docker/docker-compose
+```
+
+#### Run `docker-compose up`
+
+```bash
+$ export DOCSPELL_HEADER_VALUE="my-secret-123"
+$ docker-compose up
+```
+
+The environment variable defines a secret that is shared between some
+containers. You can define whatever you like. Please see the
+[consumedir.sh](@/docs/tools/consumedir.md#docker) docs for additional
+info.
+
+Goto `http://localhost:7880`, signup and login. When signing up, you
+choose the same name for collective and user. Then login with this
+name and the password.
+
+(Optional) Create a folder `./docs/<collective-name>` (the name you
+chose for the collective at registration) and place files in there for
+importing them.
+
+The directory contains a file `docspell.conf` that you can
+[modify](@/docs/configure/_index.md) as needed.
+
+
+### Override this setup
+
+If you want to change this setup, you can simply use your own compose
+file or add a `docker-compose.override.yml` that allows to amend
+certain configs. Look [here](https://docs.docker.com/compose/extends/)
+to find more about it.
+
+As an example, here is a `docker-compose.override.yml`:
+
+``` yaml
+version: '3.7'
+
+services:
+  consumedir:
+    volumes:
+      - importdocs:/opt/docs
+
+volumes:
+  docspell-postgres_data:
+    driver: local
+    driver_opts:
+      type: nfs4
+      o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14
+      device: ":/mnt/FreeNas/docker_vol1/docspell/postgres_data"
+
+  docspell-solr_data:
+    driver: local
+    driver_opts:
+      type: nfs4
+      o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14
+      device: ":/mnt/FreeNas/docker_vol1/docspell/solr_data"
+
+  importdocs:
+    driver: local
+    driver_opts:
+      type: nfs4
+      o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14
+      device: ":/mnt/FreeNas/archiv/gescannt/output"
+```
+
+
+### Upgrading
+
+Since [downgrading](@/docs/install/downgrading.md) is not supported,
+it is recommended to backup your database before upgrading. Should
+something not work as expected, restore the database backup and go
+back to the previous version.
+
+The latest release is always tagged with `latest`. Should you use this
+tag, then run these commands to upgrade to newer images:
+
+``` bash
+$ docker-compose down
+$ docker-compose pull
+$ docker-compose up --force-recreate --build -d
+```

website/site/content/docs/install/download_run.md

@@ -0,0 +1,121 @@
++++
+title = "Download & Run"
+weight = 22
++++
+
+You can install via zip or deb archives. Please see the
+[prerequisites](@/docs/install/prereq.md) first.
+
+## Using zip files
+
+You need to download the two files:
+
+- [docspell-restserver-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver-{{version()}}.zip)
+- [docspell-joex-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex-{{version()}}.zip)
+
+
+1. Unzip both files:
+   ``` bash
+   $ unzip docspell-*.zip
+   ```
+2. Open two terminal windows and navigate to the the directory
+   containing the zip files.
+3. Start both components executing:
+   ``` bash
+   $ ./docspell-restserver*/bin/docspell-restserver
+   ```
+   in one terminal and
+   ``` bash
+   $ ./docspell-joex*/bin/docspell-joex
+   ```
+   in the other.
+4. Point your browser to: <http://localhost:7880/app>
+5. Register a new account, sign in and try it.
+
+Note, that this setup doesn't include watching a directory nor
+fulltext search. Using zip/deb files requires to take care of the
+[prerequisites](@/docs/install/prereq.md) yourself.
+
+The provided scripts in
+[docspell-tools-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-tools-{{version()}}.zip)
+must be extracted and installed manually somewhere in your `$PATH`.
+
+## Using deb files
+
+Packages are also provided at the release page:
+
+- [docspell-restserver_{{version()}}_all.deb](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver_{{version()}}_all.deb)
+- [docspell-joex_{{version()}}_all.deb](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex_{{version()}}_all.deb)
+
+The DEB packages can be installed on Debian, or Debian based Distros:
+
+``` bash
+$ sudo dpkg -i docspell*.deb
+```
+
+Then the start scripts are in your `$PATH`. Run `docspell-restserver`
+or `docspell-joex` from a terminal window.
+
+The packages come with a systemd unit file that will be installed to
+autostart the services.
+
+The provided scripts in `docspell-tools-{{version()}}.zip` must be
+extracted and installed manually somewhere in your `$PATH`. There are
+no deb files provided.
+
+
+## Running
+
+Run the start script (in the corresponding `bin/` directory when using
+the zip files):
+
+```
+$ ./docspell-restserver*/bin/docspell-restserver
+$ ./docspell-joex*/bin/docspell-joex
+```
+
+This will startup both components using the default configuration.
+Please refer to the [configuration page](@/docs/configure/_index.md)
+for how to create a custom config file. Once you have your config
+file, simply pass it as argument to the command:
+
+```
+$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf
+$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf
+```
+
+After starting the rest server, you can reach the web application
+`http://localhost:7880/`.
+
+You should be able to create a new account and sign in. When creating
+a new account, use the same name for collective and user and then
+login with this name.
+
+## Upgrading
+
+Since [downgrading](@/docs/install/downgrading.md) is not supported,
+it is recommended to backup your database before upgrading. Should
+something not work as expected, restore the database backup and go
+back to the previous version.
+
+When using the zip or deb files, either install the new deb files via
+your package manager or download and unpack the new zip files. You
+might want to have a look at the changelog, since it is sometimes
+necessary to modify the config file.
+
+## More
+
+### Fulltext Search
+
+Fulltext search is powered by [SOLR](https://solr.apache.org). You
+need to install solr and create a core for docspell. Then cange the
+solr url for both components (restserver and joex) accordingly. See
+the relevant section in the [config
+page](@/docs/configure/_index.md#full-text-search-solr).
+
+
+### Watching a directory
+
+The [consumedir](@/docs/tools/consumedir.md) script can be used for
+this. Using systemd or something similar, it is possible to create a
+system service that runs the script in "watch mode".

website/site/content/docs/install/nix.md

@@ -1,104 +1,8 @@
 +++
-title = "Installing"
+title = "Nix / NixOS"
-weight = 20
+weight = 24
 +++
 
-# Docker
-
-There is a [docker-compose](https://docs.docker.com/compose/) setup
-available in the `/docker` folder. This setup is also taking care of
-all the necessary [prerequisites](@/docs/install/prereq.md) and
-creates a container to watch a directory for incoming files. It's only
-3 steps:
-
-1. Clone the github repository
-   ```bash
-   $ git clone https://github.com/eikek/docspell
-   ```
-   If you don't have git or don't want to clone the whole repo, use these steps instead:
-   ``` bash
-   mkdir -p docspell/docker
-   cd docspell/docker
-   wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose.yml
-   wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docspell.conf
-   wget https://raw.githubusercontent.com/eikek/docspell/master/docker/.env
-   ```
-2. Change into the `docker` directory:
-   ```bash
-   $ cd docspell/docker
-   ```
-3. Run `docker-compose up`:
-
-   ```bash
-   $ export DOCSPELL_HEADER_VALUE="my-secret-123"
-   $ docker-compose up
-   ```
-
-   The environment variable defines a secret that is shared between
-   some containers. You can define whatever you like. Please see the
-   [consumedir.sh](@/docs/tools/consumedir.md#docker) docs for
-   additional info.
-4. Goto `http://localhost:7880`, signup and login. When signing up,
-   you can choose the same name for collective and user. Then login
-   with this name and the password.
-
-5. (Optional) Create a folder `./docs/<collective-name>` (the name you
-   chose for the collective at registration) and place files in there
-   for importing them.
-
-The directory contains a file `docspell.conf` that you can
-[modify](@/docs/configure/_index.md) as needed.
-
-# Download, Unpack, Run
-
-You can install via zip or deb archives. Please see the
-[prerequisites](@/docs/install/prereq.md) first.
-
-## Using zip files
-
-You need to download the two files:
-
-- [docspell-restserver-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver-{{version()}}.zip)
-- [docspell-joex-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex-{{version()}}.zip)
-
-
-1. Unzip both files:
-   ``` bash
-   $ unzip docspell-*.zip
-   ```
-2. Open two terminal windows and navigate to the the directory
-   containing the zip files.
-3. Start both components executing:
-   ``` bash
-   $ ./docspell-restserver*/bin/docspell-restserver
-   ```
-   in one terminal and
-   ``` bash
-   $ ./docspell-joex*/bin/docspell-joex
-   ```
-   in the other.
-4. Point your browser to: <http://localhost:7880/app>
-5. Register a new account, sign in and try it.
-
-Note, that this setup doesn't include watching a directory. You can
-use the [consumedir.sh](@/docs/tools/consumedir.md) tool for this or
-use the docker variant below.
-
-## Using deb files
-
-The DEB packages can be installed on Debian, or Debian based Distros:
-
-``` bash
-$ sudo dpkg -i docspell*.deb
-```
-
-Then the start scripts are in your `$PATH`. Run `docspell-restserver`
-or `docspell-joex` from a terminal window.
-
-The packages come with a systemd unit file that will be installed to
-autostart the services.
-
-# Nix
 
 ## Install via Nix
 
@@ -159,6 +63,17 @@ selecting the most current release. For example it translates to
 is `{{version()}}`.
 
 
+
+## Upgrading
+
+Since [downgrading](@/docs/install/downgrading.md) is not supported,
+it is recommended to backup your database before upgrading. Should
+something not work as expected, restore the database backup and go
+back to the previous version.
+
+When using the provided nix setup, the `currentPkg` always points to
+the latest release. Thus it is enough to run `nix-build`.
+
 ## Docspell on NixOS {#nixos}
 
 If you are running [NixOS](https://nixos.org), there is a module

website/site/content/docs/install/prereq.md

@@ -75,7 +75,7 @@ no fulltext search.
 
 When installing manually (i.e. not via docker), just install solr and
 create a core as described in the [solr
-documentation](https://lucene.apache.org/solr/guide/8_4/installing-solr.html).
+documentation](https://solr.apache.org/guide/8_4/installing-solr.html).
 That will provide you with the connection url (the last part is the
 core name).
 

website/site/content/docs/install/quickstart.md

@@ -5,28 +5,43 @@ weight = 0
 
 To get started, here are some quick links:
 
-- Using [docker and
+- Using [docker and docker-compose](@/docs/install/docker.md). This
-  docker-compose](@/docs/install/installing.md#docker). This sets up
+  sets up everything: all prerequisites, both docspell components and
-  everything: all prerequisites, both docspell components and a
+  a container running the [consumedir.sh](@/docs/tools/consumedir.md)
-  container running the [consumedir.sh](@/docs/tools/consumedir.md)
   script to import files that are dropped in a folder.
-- [Download, Unpack and
+- [Download, Unpack and Run](@/docs/install/download_run.md). This
-  Run](@/docs/install/installing.md#download-unpack-run). This option
+  option is also very quick, but you need to check the
-  is also very quick, but you need to check the
   [prerequisites](@/docs/install/prereq.md) yourself. Database is
   already setup, but you'd need to setup SOLR (when using fulltext
   search) and install some programs for the joex component. This
   applies to the `zip` and `deb` files. The files can be downloaded
-  from the [release page](https://github.com/eikek/docspell/releases/latest).
+  from the [release
-- via the [nix package manager](@/docs/install/installing.md#nix) and/or as a [NixOS
+  page](https://github.com/eikek/docspell/releases/latest).
-  module](@/docs/install/installing.md#nixos). If you use nix/nixos, you
+- via the [nix package manager](@/docs/install/nix.md) and/or as a
-  know what to do. The linked page contains some examples.
+  [NixOS module](@/docs/install/nix.md#nixos). If you use nix/nixos,
-- [Unraid](https://www.unraid.net/): While not official, there are
+  you know what to do. The linked page contains some examples.
-  user provided [notes and unraid
+- [Unraid](https://www.unraid.net/): There are user provided [notes
+  and unraid
   templates](https://github.com/vakilando/unraid-docker-templates)
   which can get you started. Thanks for providing these!
 
-Every [component](@docs/intro/_index.md#components) (restserver, joex,
+Every [component](@/docs/intro/_index.md#components) (restserver, joex,
 consumedir) can run on different machines and multiple times. Most of
 the time running all on one machine is sufficient and also for
 simplicity, the docker-compose setup reflects this variant.
+
+While there are many different ways to run docspell, at some point all
+call docspell binaries. These accept one argument: a [config
+file](@/docs/configure/_index.md). If this is not given, the default
+is used, which gets you started on a single machine, but it is very
+likely you want to change these to match your use-case/setup.
+
+{% infobubble(mode="info", title="⚠ Please note") %}
+
+Please have a look at the [configuration
+page](@/docs/configure/_index.md) page, before making docspell
+publicly available. By default, everyone can create an account. This
+is great for trying out and using it in an internal network. But when
+opened up to the outside, it is recommended to lock this down.
+
+{% end %}

website/site/content/docs/install/running.md

@@ -1,93 +0,0 @@
-+++
-title = "Running"
-weight = 30
-+++
-
-# Running
-
-Run the start script (in the corresponding `bin/` directory when using
-the zip files):
-
-```
-$ ./docspell-restserver*/bin/docspell-restserver
-$ ./docspell-joex*/bin/docspell-joex
-```
-
-This will startup both components using the default configuration. The
-configuration should be adopted to your needs. For example, the
-database connection is configured to use a H2 database in the `/tmp`
-directory. Please refer to the [configuration
-page](@/docs/configure/_index.md) for how to create a custom config
-file. Once you have your config file, simply pass it as argument to
-the command:
-
-```
-$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf
-$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf
-```
-
-After starting the rest server, you can reach the web application at
-path `/app`, so using default values it would be
-`http://localhost:7880/app`. There also is a redirect from `/` to
-`/app`.
-
-You should be able to create a new account and sign in. Check the
-[configuration page](@/docs/configure/_index.md) to further customize
-docspell.
-
-{% infobubble(mode="info", title="⚠ Please note") %}
-
-Please have a look at the [configuration
-page](@/docs/configure/_index.md) page, before making docspell
-available in the internet. By default, everyone can create an account.
-This is great for trying out and using it in an internal network. But
-when opened up to the outside, it is recommended to lock this down.
-
-{% end %}
-
-## Memory
-
-Using the options below you can define how much memory the JVM process
-is able to use. This might be necessary to adopt depending on the
-usage scenario and configured text analysis features.
-
-Please have a look at the corresponding [configuration
-section](@/docs/configure/_index.md#memory-usage).
-
-## Options
-
-The start scripts support some options to configure the JVM. One often
-used setting is the maximum heap size of the JVM. By default, java
-determines it based on properties of the current machine. You can
-specify it by given java startup options to the command:
-
-```
-$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf
-```
-
-This would limit the maximum heap to 1GB. The double slash separates
-internal options and the arguments to the program. Another frequently
-used option is to change the default temp directory. Usually it is
-`/tmp`, but it may be desired to have a dedicated temp directory,
-which can be configured:
-
-```
-$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf
-```
-
-The command:
-
-```
-$ ./docspell-restserver*/bin/docspell-restserver -h
-```
-
-gives an overview of supported options.
-
-It is recommended to run joex with the G1GC enabled. If you use java8,
-you need to add an option to use G1GC (`-XX:+UseG1GC`), for java11
-this is not necessary (but doesn't hurt either). This could look like
-this:
-
-```
-./docspell-joex-{{version()}}/bin/docspell-joex -J-Xmx1596M -J-XX:+UseG1GC -- /path/to/joex.conf
-```

website/site/content/docs/install/upgrading.md

@@ -1,35 +0,0 @@
-+++
-title = "Upgrading"
-weight = 35
-+++
-
-Upgrading docspell requires to download the newer binaries and
-re-installing them depending on your setup.
-
-The database is migrated on application start automatically.
-
-Since [downgrading](@/docs/install/downgrading.md) is not supported,
-it is recommended to backup your database before upgrading. Should
-something not work as expected, restore the database backup and go
-back to the previous version.
-
-# Docker-Compose
-
-When using the provided `docker-compose` setup, you only need to pull
-the new images. The latest release is always tagged with `-LATEST`.
-
-``` bash
-$ docker-compose down
-$ docker-compose pull
-$ docker-compose up --force-recreate --build -d
-```
-
-# Zip / Deb Files
-
-When using the zip or deb files, either install the new deb files via
-your package manager or download and unpack the new zip files.
-
-# Nix
-
-When using the provided nix setup, the `currentPkg` always points to
-the latest release. Thus it is enough to run `nix-build`.

website/site/content/docs/intro/_index.md

@@ -85,7 +85,7 @@ database. And it is also recommended (though not strictly required),
 that all components can reach each other.
 
 The fulltext search index is another separate component, where
-currently only [SOLR](https://lucene.apache.org/solr) is supported.
+currently only [SOLR](https://solr.apache.org) is supported.
 Fulltext search is optional, so the SOLR component is not required if
 docspell is run without fulltext search support.
 

website/site/content/docs/webapp/finding.md

@@ -161,10 +161,10 @@ unless one of the following is true:
 ## The Query
 
 The query string for full text search is very powerful. Docspell
-currently supports [Apache SOLR](https://lucene.apache.org/solr/) as
+currently supports [Apache SOLR](https://solr.apache.org/) as
 full text search backend, so you may want to have a look at their
 [documentation on query
-syntax](https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing)
+syntax](https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing)
 for a in depth guide.
 
 - Wildcards: `?` matches any single character, `*` matches zero or