diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000..2bc9a86e --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,22 @@ +name: Publish Docker Images +on: + release: + types: [ published ] +jobs: + docker-images: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - name: Set current version + run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV + - name: Build Docker Images (${{ env.DOCSPELL_VERSION }}) + run: ./docker/dockerfiles/build.sh ${{ env.DOCSPELL_VERSION }} + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Publish Images (${{ env.DOCSPELL_VERSION }}) + run: ./docker/dockerfiles/push.sh ${{ env.DOCSPELL_VERSION }} diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml new file mode 100644 index 00000000..908c87a1 --- /dev/null +++ b/.github/workflows/release-nightly.yml @@ -0,0 +1,52 @@ +name: Release Nightly +on: + push: + branches: + - "master" +jobs: + release-nightly: + runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + java: [ 'openjdk@1.11' ] + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v10 + with: + java-version: ${{ matrix.java }} + - uses: jorelali/setup-elm@v3 + with: + elm-version: 0.19.1 + - uses: bahmutov/npm-install@v1 + - name: Coursier cache + uses: coursier/cache-action@v6 + - name: Set current version + run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV + - name: sbt ci ${{ github.ref }} + run: sbt ci + - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }}) + run: sbt make-pkg + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "nightly" + prerelease: true + title: "Docspell Nightly" + files: | + modules/restserver/target/docspell-restserver_${{ env.DOCSPELL_VERSION }}_all.deb + modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip + modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb + modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip + tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip + - name: Build Docker Images (${{ env.DOCSPELL_VERSION }}) + run: ./docker/dockerfiles/build.sh ${{ env.DOCSPELL_VERSION }} + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Publish Images (${{ env.DOCSPELL_VERSION }}) + run: ./docker/dockerfiles/push.sh ${{ env.DOCSPELL_VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..2245f47b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release +on: + push: + tags: + - 'v*' +jobs: + release: + runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + java: [ 'openjdk@1.11' ] + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v10 + with: + java-version: ${{ matrix.java }} + - uses: jorelali/setup-elm@v3 + with: + elm-version: 0.19.1 + - uses: bahmutov/npm-install@v1 + - name: Coursier cache + uses: coursier/cache-action@v6 + - name: Set current version + run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV + - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }}) + run: sbt make-pkg + - uses: meeDamian/github-release@2.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + name: Docspell ${{ env.DOCSPELL_VERSION }} + gzip: false + draft: true + prerelease: false + allow_override: false + files: > + modules/restserver/target/docspell-restserver_${{ env.DOCSPELL_VERSION }}_all.deb + modules/restserver/target/universal/docspell-restserver-${{ env.DOCSPELL_VERSION }}.zip + modules/joex/target/docspell-joex_${{ env.DOCSPELL_VERSION }}_all.deb + modules/joex/target/universal/docspell-joex-${{ env.DOCSPELL_VERSION }}.zip + tools/target/docspell-tools-${{ env.DOCSPELL_VERSION }}.zip diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml new file mode 100644 index 00000000..410ee5e4 --- /dev/null +++ b/.github/workflows/website.yml @@ -0,0 +1,26 @@ +name: Publish Website +on: + push: + branches: + - "current-docs" +jobs: + publish-website: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - name: Set current version + run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV + - uses: jorelali/setup-elm@v3 + with: + elm-version: 0.19.1 + - uses: cachix/install-nix-action@v13 + with: + nix_path: nixpkgs=channel:nixos-20.09 + - name: Print nixpkgs version + run: nix-instantiate --eval -E '(import {}).lib.version' + - name: Build website (${{ env.DOCSPELL_VERSION }}) + run: nix-shell website/shell.nix --run "sbt make-website" + - name: Publish website (${{ env.DOCSPELL_VERSION }}) + run: sbt publish-website diff --git a/Changelog.md b/Changelog.md index 79f1d1f3..41940f73 100644 --- a/Changelog.md +++ b/Changelog.md @@ -895,10 +895,9 @@ improvements for processing files. - Adds the [full-text search](https://docspell.org/docs/webapp/finding/#full-text-search) feature (see #69). It requires a separate - [Solr](https://lucene.apache.org/solr) instance. Items can be - searched by documents contents and item/file names. It is possible - to use full-text search to further confine the results via the - search menu. + [Solr](https://solr.apache.org) instance. Items can be searched by + documents contents and item/file names. It is possible to use + full-text search to further confine the results via the search menu. - Fixes column types for item date and due-date for MariaDB (see #158) and adds an upper limit for due-dates (which is configurable). - Fixes a bug when cancelling jobs. Stuck jobs were only removed from diff --git a/Contributing.md b/Contributing.md index 40d71929..a470ec44 100644 --- a/Contributing.md +++ b/Contributing.md @@ -63,5 +63,5 @@ nice functional language that compiles to javascript. The frontend is included in the http/restserver component. The CSS is provided by [TailwindCSS](https://tailwindcss.com/).. -The [development](https://docspell.org/docs/dev/building/) page -contains some tips to get started. +The [development](https://docspell.org/docs/dev) pages contains some +tips to get started. diff --git a/build.sbt b/build.sbt index f330ee2f..0422c9ab 100644 --- a/build.sbt +++ b/build.sbt @@ -776,8 +776,8 @@ addCommandAlias( "make", ";set webapp/elmCompileMode := ElmCompileMode.Production; set webapp/stylesMode := StylesMode.Prod ;root/openapiCodegen ;root/Test/compile" ) -addCommandAlias("make-zip", ";restserver/universal:packageBin ;joex/universal:packageBin") -addCommandAlias("make-deb", ";restserver/debian:packageBin ;joex/debian:packageBin") +addCommandAlias("make-zip", ";restserver/Universal/packageBin ;joex/Universal/packageBin") +addCommandAlias("make-deb", ";restserver/Debian/packageBin ;joex/Debian/packageBin") addCommandAlias("make-tools", ";root/toolsPackage") addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb ;make-tools") @@ -787,3 +787,5 @@ addCommandAlias( "scalafmtSbtCheck; scalafmtCheckAll; Compile/scalafix --check; Test/scalafix --check" ) addCommandAlias("fix", "Compile/scalafix; Test/scalafix; scalafmtSbt; scalafmtAll") +addCommandAlias("make-website", ";website/clean ;website/zolaBuild ;website/zolaCheck") +addCommandAlias("publish-website", "website/publishToGitHubPages") diff --git a/docker/docker-compose/.env b/docker/docker-compose/.env new file mode 100644 index 00000000..7f5e39ae --- /dev/null +++ b/docker/docker-compose/.env @@ -0,0 +1,2 @@ +TZ=Europe/Berlin +DOCSPELL_HEADER_VALUE=none diff --git a/docker/docker-compose/docker-compose.yml b/docker/docker-compose/docker-compose.yml new file mode 100644 index 00000000..25a6139b --- /dev/null +++ b/docker/docker-compose/docker-compose.yml @@ -0,0 +1,80 @@ +version: '3.8' +services: + + restserver: + image: docspell/restserver:latest + container_name: docspell-restserver + command: + - /opt/docspell.conf + restart: unless-stopped + ports: + - "7880:7880" + volumes: + - ./docspell.conf:/opt/docspell.conf + env_file: ./.env + depends_on: + - solr + + joex: + image: docspell/joex:latest + container_name: docspell-joex + command: + - /opt/docspell.conf + restart: unless-stopped + env_file: ./.env + ports: + - "7878:7878" + volumes: + - ./docspell.conf:/opt/docspell.conf + depends_on: + - solr + + consumedir: + image: docspell/tools:latest + container_name: docspell-consumedir + command: + - ds-consumedir + - "-vmdi" + - "--path" + - "/opt/docs" + - "--iheader" + - "Docspell-Integration:$DOCSPELL_HEADER_VALUE" + - "http://docspell-restserver:7880/api/v1/open/integration/item" + restart: unless-stopped + env_file: ./.env + volumes: + - ./docs:/opt/docs + depends_on: + - restserver + + db: + image: postgres:13.3 + container_name: postgres_db + restart: unless-stopped + volumes: + - docspell-postgres_data:/var/lib/postgresql/data/ + environment: + - POSTGRES_USER=dbuser + - POSTGRES_PASSWORD=dbpass + - POSTGRES_DB=dbname + + solr: + image: solr:8 + container_name: docspell-solr + restart: unless-stopped + volumes: + - docspell-solr_data:/var/solr + command: + - solr-precreate + - docspell + healthcheck: + test: ["CMD", "curl", "f", "http://localhost:8983/solr/docspell/admin/ping"] + interval: 1m + timeout: 10s + retries: 2 + start_period: 30s + + +volumes: + docspell-postgres_data: + docspell-solr_data: diff --git a/docker/docker-compose/docspell.conf b/docker/docker-compose/docspell.conf new file mode 100644 index 00000000..03feea5e --- /dev/null +++ b/docker/docker-compose/docspell.conf @@ -0,0 +1,83 @@ +# This is the configuration file for docspell. It contains two main +# namespaces: docspell.server and docspell.joex. Each provide the +# config for the respective component. +# +# They can be moved to different files, if necessary. For this example +# though, both components are configured in this single file. +# +# Below are only some settings that differ from the default. Please +# see https://docspell.org/docs/configure/#default-config for all +# options and their documentation. This page provides more +# information about the important config options. +# +# Note: this docker-compose setup is an example to get started. It +# sets up one rest-server, one joex, a postgres database and a solr – +# all on the same machine. + +# Define settings that are used in multiple places: +db_url="jdbc:postgresql://db:5432/dbname" +db_user="dbuser" +db_pass="dbpass" +solr_url="http://docspell-solr:8983/docspell" + + +# This configures the restserver +docspell.server { + base-url = "http://localhost:7880" + bind { + address = "0.0.0.0" + } + integration-endpoint { + enabled = true + http-header { + enabled = true + header-value = ${?DOCSPELL_HEADER_VALUE} + } + } + # Configuration of the full-text search engine. + full-text-search { + enabled = true + solr = { + url = ${solr_url} + } + } + backend { + jdbc { + url = ${db_url} + user = ${db_user} + password = ${db_pass} + } + } +} + + +# This configures joex +# +# Note to joex: It is currently setup for one instance. Should you +# want to scale joex instance up (maybe to help processing a batch of +# files), there are two options: +# +# - look at https://github.com/eikek/docspell/pull/552 to elastically +# start and stop joex instances via docker-compose +# - set pool-size to some higher number; this requires to restart joex +# +docspell.joex { + base-url = "http://docspell-joex:7878" + bind { + address = "0.0.0.0" + } + jdbc { + url = ${db_url} + user = ${db_user} + password = ${db_pass} + } + full-text-search { + enabled = true + solr = { + url = ${solr_url} + } + } + scheduler { + pool-size = 1 + } +} diff --git a/docker/dockerfiles/build.sh b/docker/dockerfiles/build.sh new file mode 100755 index 00000000..d278a73b --- /dev/null +++ b/docker/dockerfiles/build.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +if [ -z "$1" ]; then + echo "Please specify a version" + exit 1 +fi + +version="$1" +if [[ $version == v* ]]; then + version="${version:1}" +fi + +set -e +cd "$(dirname "$0")" + +if [[ $version == *SNAPSHOT* ]]; then + echo ">>>> Building nightly images for $version <<<<<" + url_base="https://github.com/eikek/docspell/releases/download/nightly" + + echo "============ Building Tools ============" + docker build \ + --build-arg tools_url="$url_base/docspell-tools-$version.zip" \ + --tag docspell/tools:nightly \ + -f tools.dockerfile . + + echo "============ Building Restserver ============" + docker build \ + --build-arg restserver_url="$url_base/docspell-restserver-$version.zip" \ + --tag docspell/restserver:nightly \ + -f restserver.dockerfile . + + echo "============ Building Joex ============" + docker build \ + --build-arg joex_url="$url_base/docspell-joex-$version.zip" \ + --tag docspell/joex:nightly \ + -f joex.dockerfile . +else + echo ">>>> Building release images for $version <<<<<" + echo "============ Building Tools ============" + docker build \ + --build-arg version=$version \ + --tag docspell/tools:v$version \ + --tag docspell/tools:latest \ + -f tools.dockerfile . + + echo "============ Building Restserver ============" + docker build \ + --build-arg version=$version \ + --tag docspell/restserver:v$version \ + --tag docspell/restserver:latest \ + -f restserver.dockerfile . + + echo "============ Building Joex ============" + docker build \ + --build-arg version=$version \ + --tag docspell/joex:v$version \ + --tag docspell/joex:latest \ + -f joex.dockerfile . +fi diff --git a/docker/dockerfiles/joex-entrypoint.sh b/docker/dockerfiles/joex-entrypoint.sh new file mode 100755 index 00000000..3ed3df29 --- /dev/null +++ b/docker/dockerfiles/joex-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "Starting unoconv listener" +unoconv -l & + +/opt/docspell-joex/bin/docspell-joex "$@" diff --git a/docker/dockerfiles/joex.dockerfile b/docker/dockerfiles/joex.dockerfile new file mode 100644 index 00000000..5a882732 --- /dev/null +++ b/docker/dockerfiles/joex.dockerfile @@ -0,0 +1,68 @@ +FROM alpine:latest + +ARG version= +ARG joex_url= +ARG UNO_URL=https://raw.githubusercontent.com/unoconv/unoconv/0.9.0/unoconv + +ENV JAVA_OPTS="-Xmx1536M" + +RUN apk add --no-cache openjdk11 \ + tzdata \ + bash \ + curl \ + ghostscript \ + tesseract-ocr \ + tesseract-ocr-data-deu \ + tesseract-ocr-data-fra \ + tesseract-ocr-data-ita \ + tesseract-ocr-data-spa \ + tesseract-ocr-data-por \ + tesseract-ocr-data-ces \ + tesseract-ocr-data-nld \ + tesseract-ocr-data-dan \ + tesseract-ocr-data-fin \ + tesseract-ocr-data-nor \ + tesseract-ocr-data-swe \ + tesseract-ocr-data-rus \ + tesseract-ocr-data-ron \ + tesseract-ocr-data-lav \ + unpaper \ + wkhtmltopdf \ + libreoffice \ + ttf-droid-nonlatin \ + ttf-droid \ + ttf-dejavu \ + ttf-freefont \ + ttf-liberation \ + libxml2-dev \ + libxslt-dev \ + pngquant \ + zlib-dev \ + g++ \ + qpdf \ + py3-pip \ + python3-dev \ + libffi-dev\ + qpdf-dev \ + openssl-dev \ + ocrmypdf \ + && pip3 install --upgrade pip \ + && pip3 install ocrmypdf \ + && curl -Ls $UNO_URL -o /usr/local/bin/unoconv \ + && chmod +x /usr/local/bin/unoconv \ + && apk del curl libxml2-dev libxslt-dev zlib-dev g++ python3-dev py3-pip libffi-dev qpdf-dev openssl-dev \ + && ln -s /usr/bin/python3 /usr/bin/python + +WORKDIR /opt +RUN wget ${joex_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-joex-$version.zip} && \ + unzip docspell-joex-*.zip && \ + rm docspell-joex-*.zip && \ + ln -snf docspell-joex-* docspell-joex + +COPY joex-entrypoint.sh /opt/joex-entrypoint.sh + +ENTRYPOINT ["/opt/joex-entrypoint.sh"] +EXPOSE 7878 + +HEALTHCHECK --interval=1m --timeout=10s --retries=2 --start-period=10s \ + CMD pgrep -f joex/lib diff --git a/docker/dockerfiles/push.sh b/docker/dockerfiles/push.sh new file mode 100755 index 00000000..3d72b3b9 --- /dev/null +++ b/docker/dockerfiles/push.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + + +if [ -z "$1" ]; then + echo "Please specify a version" + exit 1 +fi + +version="$1" +if [[ $version == v* ]]; then + version="${version:1}" +fi + +set -e +cd "$(dirname "$0")" + +if [[ $version == *SNAPSHOT* ]]; then + echo "============ Push Tools ============" + docker push docspell/tools:nightly + + echo "============ Push Restserver ============" + docker push docspell/restserver:nightly + + echo "============ Push Joex ============" + docker push docspell/joex:nightly +else + echo "============ Push Tools ============" + docker push docspell/tools:v$version + docker push docspell/tools:latest + + echo "============ Push Restserver ============" + docker push docspell/restserver:v$version + docker push docspell/restserver:latest + + echo "============ Push Joex ============" + docker push docspell/joex:v$version + docker push docspell/joex:latest +fi diff --git a/docker/dockerfiles/restserver.dockerfile b/docker/dockerfiles/restserver.dockerfile new file mode 100644 index 00000000..78afeec1 --- /dev/null +++ b/docker/dockerfiles/restserver.dockerfile @@ -0,0 +1,18 @@ +FROM alpine:latest + +ARG version= +ARG restserver_url= + +RUN apk add --no-cache openjdk11 bash tzdata + +WORKDIR /opt +RUN wget ${restserver_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-restserver-$version.zip} && \ + unzip docspell-restserver-*.zip && \ + rm docspell-restserver-*.zip && \ + ln -snf docspell-restserver-* docspell-restserver + +ENTRYPOINT ["/opt/docspell-restserver/bin/docspell-restserver"] +EXPOSE 7880 + +HEALTHCHECK --interval=1m --timeout=10s --retries=2 --start-period=30s \ + CMD wget --spider http://localhost:7880 diff --git a/docker/dockerfiles/tools.dockerfile b/docker/dockerfiles/tools.dockerfile new file mode 100644 index 00000000..cb6b85ea --- /dev/null +++ b/docker/dockerfiles/tools.dockerfile @@ -0,0 +1,35 @@ +FROM alpine:latest + +# Builds an image where all scripts in tools/ are in PATH. There are +# no assumptions what script to run, so there are no CMD or +# ENTRYPOINTS defined. +# +# The scripts are named is in tools/ only prefixed by `ds-` +# +# Run the export-files script, for example: +# +# docker run -e DS_USER=demo -e DS_PASS=test docspell/tools:dev ds-export-files "http://localhost" . +# +# The build requires to either specify a version build argument or a +# tools_url build argument. If a tools_url argument is given, then +# this url is used to download the tools zip file. Otherwise the +# version argument is used to download from github. + +LABEL maintainer="eikek0 " + +ARG version= +ARG tools_url= + +RUN apk add --no-cache curl bash inotify-tools jq sqlite + +WORKDIR /opt +RUN wget ${tools_url:-https://github.com/eikek/docspell/releases/download/v$version/docspell-tools-$version.zip} && \ + unzip docspell-tools-*.zip && \ + rm docspell-tools-*.zip + +RUN bash -c 'while read f; do \ + target="ds-$(basename "$f" ".sh")"; \ + echo "Installing $f -> $target"; \ + cp "$f" "/usr/local/bin/$target"; \ + chmod 755 "/usr/local/bin/$target"; \ + done < <(find /opt/docspell-tools-* -name "*.sh" -mindepth 2 -not -path "*webextension*")' diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 78e80144..97ecc84c 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -575,7 +575,7 @@ docspell.joex { log-verbose = false # The defType parameter to lucene that defines the parser to # use. You might want to try "edismax" or look here: - # https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing + # https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing def-type = "lucene" # The default combiner for tokens. One of {AND, OR}. q-op = "OR" diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index e7ec39f8..52cf2991 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -145,7 +145,7 @@ docspell.server { log-verbose = false # The defType parameter to lucene that defines the parser to # use. You might want to try "edismax" or look here: - # https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing + # https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing def-type = "lucene" # The default combiner for tokens. One of {AND, OR}. q-op = "OR" diff --git a/nix/module-joex.nix b/nix/module-joex.nix index 749b35a4..e509ff97 100644 --- a/nix/module-joex.nix +++ b/nix/module-joex.nix @@ -1268,7 +1268,7 @@ in { description = '' The defType parameter to lucene that defines the parser to use. You might want to try "edismax" or look here: - https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing + https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing ''; }; q-op = mkOption { diff --git a/nix/module-server.nix b/nix/module-server.nix index d079f778..0808fba6 100644 --- a/nix/module-server.nix +++ b/nix/module-server.nix @@ -397,7 +397,7 @@ in { description = '' The defType parameter to lucene that defines the parser to use. You might want to try "edismax" or look here: - https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing + https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing ''; }; q-op = mkOption { diff --git a/tools/consumedir/consumedir.sh b/tools/consumedir/consumedir.sh index e594ddc5..22f336d1 100755 --- a/tools/consumedir/consumedir.sh +++ b/tools/consumedir/consumedir.sh @@ -22,7 +22,7 @@ if [[ ${PIPESTATUS[0]} -ne 4 ]]; then fi OPTIONS=omhdp:vrmi -LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader: +LONGOPTS=once,distinct,help,delete,path:,verbose,recursive,dry,integration,iuser:,iheader:,poll: ! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@") if [[ ${PIPESTATUS[0]} -ne 0 ]]; then @@ -36,7 +36,7 @@ eval set -- "$PARSED" declare -a watchdir help=n verbose=n delete=n once=n distinct=n recursive=n dryrun=n -integration=n iuser="" iheader="" +integration=n iuser="" iheader="" poll="" while true; do case "$1" in -h|--help) @@ -84,6 +84,10 @@ while true; do iheader="$2" shift 2 ;; + --poll) + poll="$2" + shift 2 + ;; --) shift break @@ -108,6 +112,8 @@ showUsage() { echo " -h | --help Prints this help text. (value: $help)" echo " -m | --distinct Optional. Upload only if the file doesn't already exist. (value: $distinct)" echo " -o | --once Instead of watching, upload all files in that dir. (value: $once)" + echo " --poll Run the script periodically instead of watching a directory. This can be" + echo " used if watching via inotify is not possible." echo " -r | --recursive Traverse the directory(ies) recursively (value: $recursive)" echo " -i | --integration Upload to the integration endpoint. It implies -r. This puts the script in" echo " a different mode, where the first subdirectory of any given starting point" @@ -134,7 +140,7 @@ showUsage() { echo "$0 --path ~/Downloads -m -dv --once http://localhost:7880/api/v1/open/upload/item/abcde-12345-abcde-12345" echo "" echo "Example: Integration Endpoint" - echo "$0 -i -iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item" + echo "$0 -i --iheader 'Docspell-Integration:test123' -m -p ~/Downloads/ http://localhost:7880/api/v1/open/integration/item" echo "" } @@ -355,11 +361,7 @@ checkSetup() { done } - -# warn if something seems not correctly configured -checkSetup - -if [ "$once" = "y" ]; then +runOnce() { info "Uploading all files (except hidden) in '$watchdir'." MD="-maxdepth 1" if [ "$recursive" = "y" ]; then @@ -370,20 +372,37 @@ if [ "$once" = "y" ]; then process "$file" "$dir" done done +} + + +# warn if something seems not correctly configured +checkSetup + +if [ "$once" = "y" ]; then + runOnce else REC="" if [ "$recursive" = "y" ]; then REC="-r" fi - $INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" | - while read pathfile; do - if [[ "$(basename "$pathfile")" != .* ]]; then - dir=$(findDir "$pathfile") - trace "The file '$pathfile' appeared below '$dir'" - sleep 1 - process "$(realpath "$pathfile")" "$dir" - else - trace "Skip hidden file $(realpath "$pathfile")" - fi + if [ -z "$poll" ]; then + $INOTIFY_CMD $REC -m --format '%w%f' -e close_write -e moved_to "${watchdir[@]}" | + while read pathfile; do + if [[ "$(basename "$pathfile")" != .* ]]; then + dir=$(findDir "$pathfile") + trace "The file '$pathfile' appeared below '$dir'" + sleep 1 + process "$(realpath "$pathfile")" "$dir" + else + trace "Skip hidden file $(realpath "$pathfile")" + fi + done + else + echo "Running in polling mode: ${poll}s" + while [ : ] + do + runOnce + sleep $poll done + fi fi diff --git a/website/elm/GetStarted.elm b/website/elm/GetStarted.elm index dd5dee4a..3779c0ff 100644 --- a/website/elm/GetStarted.elm +++ b/website/elm/GetStarted.elm @@ -20,7 +20,7 @@ getStarted version = Alternatively, [download](https://github.com/eikek/docspell/archive/master.zip) the sources and extract the zip file. 2. Change into the `docker` directory: ```bash - $ cd docspell/docker + $ cd docspell/docker/docker-compose ``` 3. Run `docker-compose up`: @@ -34,7 +34,7 @@ getStarted version = [`consumedir.sh`](doc/tools/consumedir#docker) docs for additional info. 4. Goto , signup and login. When signing up, - you can choose the same name for collective and user. Then login + choose the same name for collective and user. Then login with this name and the password. 5. (Optional) Create a folder `./docs/` (the name you diff --git a/website/site/content/docs/configure/_index.md b/website/site/content/docs/configure/_index.md index 6af41559..8438f196 100644 --- a/website/site/content/docs/configure/_index.md +++ b/website/site/content/docs/configure/_index.md @@ -7,12 +7,63 @@ weight = 40 mktoc = true +++ -Docspell's executable can take one argument – a configuration file. If -that is not given, the defaults are used. The config file overrides -default values, so only values that differ from the defaults are -necessary. +Docspell's executables (restserver and joex) can take one argument – a +configuration file. If that is not given, the defaults are used. The +config file overrides default values, so only values that differ from +the defaults are necessary. The complete default options and their +documentation is at the end of this page. + +Besides the config file, another way is to provide individual settings +via key-value pairs to the executable by the `-D` option. For example +to override only `base-url` you could add the argument +`-Ddocspell.server.base-url=…` to the command. Multiple options are +possible. For more than few values this is very tedious, obviously, so +the recommended way is to maintain a config file. If these options +*and* a file is provded, then any setting given via the `-D…` option +overrides the same setting from the config file. + +# File Format + +The format of the configuration files can be +[HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation), +JSON or what this [config +library](https://github.com/lightbend/config) understands. The default +values below are in HOCON format, which is recommended, since it +allows comments and has some [advanced +features](https://github.com/lightbend/config#features-of-hocon). +Please refer to their documentation for more on this. + +A short description (please see the links for better understanding): +The config consists of key-value pairs and can be written in a +JSON-like format (called HOCON). Keys are organized in trees, and a +key defines a full path into the tree. There are two ways: + +``` +a.b.c.d=15 +``` + +or + +``` +a { + b { + c { + d = 15 + } + } +} +``` + +Both are exactly the same and these forms are both used at the same +time. Usually the braces approach is used to group some more settings, +for better readability. + +Strings that contain "not-so-common" characters should be enclosed in +quotes. It is possible to define values at the top of the file and +reuse them on different locations via the `${full.path.to.key}` +syntax. When using these variables, they *must not* be enclosed in +quotes. -This applies to the restserver and the joex as well. # Important Config Options @@ -95,7 +146,7 @@ endpoint. ## Full-Text Search: SOLR -[Apache SOLR](https://lucene.apache.org/solr) is used to provide the +[Apache SOLR](https://solr.apache.org) is used to provide the full-text search. Both docspell components must provide the same connection setup. This is defined in the `full-text-search.solr` subsection: @@ -120,7 +171,7 @@ to `true`. When installing docspell manually, just install solr and create a core as described in the [solr -documentation](https://lucene.apache.org/solr/guide/8_4/installing-solr.html). +documentation](https://solr.apache.org/guide/8_4/installing-solr.html). That will provide you with the connection url (the last part is the core name). @@ -143,7 +194,7 @@ by a joex instance eventually. Using this endpoint, the index will be re-created. This is sometimes necessary, for example if you upgrade SOLR or delete the core to provide a new one (see -[here](https://lucene.apache.org/solr/guide/8_4/reindexing.html) for +[here](https://solr.apache.org/guide/8_4/reindexing.html) for details). Note that a collective can also re-index their data using a similiar endpoint; but this is only deleting their data and doesn't do a full re-index. @@ -394,53 +445,50 @@ The restserver component is very lightweight, here you can use defaults. -# File Format +# JVM Options -The format of the configuration files can be -[HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation), -JSON or whatever the used [config -library](https://github.com/lightbend/config) understands. The default -values below are in HOCON format, which is recommended, since it -allows comments and has some [advanced -features](https://github.com/lightbend/config#features-of-hocon). -Please refer to their documentation for more on this. - -A short description (please see the links for better understanding): -The config consists of key-value pairs and can be written in a -JSON-like format (called HOCON). Keys are organized in trees, and a -key defines a full path into the tree. There are two ways: +The start scripts support some options to configure the JVM. One often +used setting is the maximum heap size of the JVM. By default, java +determines it based on properties of the current machine. You can +specify it by given java startup options to the command: ``` -a.b.c.d=15 +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf ``` -or +This would limit the maximum heap to 1GB. The double slash separates +internal options and the arguments to the program. Another frequently +used option is to change the default temp directory. Usually it is +`/tmp`, but it may be desired to have a dedicated temp directory, +which can be configured: ``` -a { - b { - c { - d = 15 - } - } -} +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf ``` -Both are exactly the same and these forms are both used at the same -time. Usually the braces approach is used to group some more settings, -for better readability. +The command: +``` +$ ./docspell-restserver*/bin/docspell-restserver -h +``` -# Default Config -## Rest Server +gives an overview of supported options. -{{ incl_conf(path="templates/shortcodes/server.conf") }} +It is recommended to run joex with the G1GC enabled. If you use java8, +you need to add an option to use G1GC (`-XX:+UseG1GC`), for java11 +this is not necessary (but doesn't hurt either). This could look like +this: +``` +./docspell-joex-{{version()}}/bin/docspell-joex -J-Xmx1596M -J-XX:+UseG1GC -- /path/to/joex.conf +``` -## Joex +Using these options you can define how much memory the JVM process is +able to use. This might be necessary to adopt depending on the usage +scenario and configured text analysis features. +Please have a look at the corresponding [section](@/docs/configure/_index.md#memory-usage). -{{ incl_conf(path="templates/shortcodes/joex.conf") }} # Logging @@ -480,3 +528,15 @@ The `` means, that only log statements with level "INFO" will be printed. But the `` above says, that for loggers with name "docspell" statements with level "DEBUG" will be printed, too. + + +# Default Config +## Rest Server + +{{ incl_conf(path="templates/shortcodes/server.conf") }} + + +## Joex + + +{{ incl_conf(path="templates/shortcodes/joex.conf") }} diff --git a/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md b/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md index e979eeae..335dccac 100644 --- a/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md +++ b/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md @@ -20,7 +20,7 @@ work (just the fulltext search is then not working). # Considered Options -* [Apache SOLR](https://lucene.apache.org/solr) +* [Apache SOLR](https://solr.apache.org) * [ElasticSearch](https://www.elastic.co/elasticsearch/) * [PostgreSQL](https://www.postgresql.org/docs/12/textsearch.html) * All of them or a subset diff --git a/website/site/content/docs/dev/building.md b/website/site/content/docs/dev/building.md index 395ae3af..2a52416f 100644 --- a/website/site/content/docs/dev/building.md +++ b/website/site/content/docs/dev/building.md @@ -8,7 +8,8 @@ You must install [sbt](https://scala-sbt.org), [nodejs](https://www.npmjs.com/get-npm) (for the `npm` command) and [Elm](https://elm-lang.org). -Clone the sources and run: +Clone the sources, `cd` into the new directory and run `sbt`. This +drops you in the sbt prompt. Then these tasks can be run: - `make` to compile all sources (Elm + Scala) - `make-zip` to create zip packages diff --git a/website/site/content/docs/dev/development.md b/website/site/content/docs/dev/development.md index 4e66bc61..6044951d 100644 --- a/website/site/content/docs/dev/development.md +++ b/website/site/content/docs/dev/development.md @@ -139,9 +139,53 @@ sets more memory for the vm): ``` bash export QEMU_OPTS="-m 2048" +export QEMU_NET_OPTS "hostfwd=tcp::7880-:7880" ./result/bin/run-docspelltest-vm ``` +# Release + +The CI and making a release is done via [github +actions](https://docs.github.com/en/actions). The workflow is roughly +like this: + +- each PR is only merged if the `sbt ci` task returns successfully. + This is ensured by the `ci.yml` workflow that triggers on each pull + request +- each commit to the `master` branch is also going through `sbt ci` + and then a prerelease is created. The tag `nightly` is used to point + to the latest commit in `master`. Note, that this is [discouraged by + git](https://git-scm.com/docs/git-tag#_on_re_tagging), but github + doesn't allow to create a release without a tag. So this tag moves + (and is not really a tag then…). After the prerelease is created, + the docker images are built and pushed to docker hub into the + [docspell](https://hub.docker.com/u/docspell) organization. The + docker images are also tagged with `nightly` at docker hub. This is + all done via the `realease-nightly.yml` workflow. +- A stable release is started by pushing a tag with pattern `v*` to + github. This triggers the `release.yml` workflow which builds the + packages and creates a release in *draft mode*. The `sbt ci` task + *is not* run, because it is meant to only release commits already in + the `master` branch. After this completes, the release notes need to + be added manually and then the release must be published at github. + This then triggers the `docker-images.yml` workflow, which builds + the corresponding docker images and pushes them to docker hub. The + docker images are tagged with the exact version and the `latest` tag + is moved to the new images. Another manual step is to set the branch + `current-docs` to its new state and push it to github. This will + trigger a build+publish of the website. +- Publishing the website happens automatically on each push to the + branch `current-docs`. Changes to the current website must be based + on this branch. + +Some notes: I wanted a 2/3 step process when doing a stable release, +to be able to add release notes manually (I don't want this to be +automated right now) and to do some testing with the packages before +publishing the release. However, for the nightly releases, this +doesn't matter - everything must be automated here obviously. I also +wanted the docker images to be built from the exact same artifacts +that have been released at github (in contrast to being built again). + # Background Info diff --git a/website/site/content/docs/features/_index.md b/website/site/content/docs/features/_index.md index cf213621..5a3497d5 100644 --- a/website/site/content/docs/features/_index.md +++ b/website/site/content/docs/features/_index.md @@ -13,7 +13,7 @@ description = "A list of features and limitations." - Handle multiple documents as one unit - OCR using [tesseract](https://github.com/tesseract-ocr/tesseract) - [Full-Text Search](@/docs/webapp/finding.md#full-text-search) based - on [Apache SOLR](https://lucene.apache.org/solr) + on [Apache SOLR](https://solr.apache.org) - Conversion to PDF: all files are converted into a PDF file. PDFs with only images (as often returned from scanners) are converted into searchable PDF/A pdfs. diff --git a/website/site/content/docs/install/docker.md b/website/site/content/docs/install/docker.md new file mode 100644 index 00000000..db222765 --- /dev/null +++ b/website/site/content/docs/install/docker.md @@ -0,0 +1,242 @@ ++++ +title = "Docker" +weight = 20 ++++ + +## Docker Images + +The docker images are at +[hub.docker.com](https://hub.docker.com/u/docspell). The `latest` tag +always points to the latest *release*. The releases are also tagged +with their respective version number. Additionally, there are images +tagged with `nightly` which are built from the `master` branch. +Therefore the `nightly` packages should be used with care, because +things might break in between. But they are useful for trying out +something. + +There are images for all components that are available from the github +release page. The images contain all the necessary +[prerequisites](@/docs/install/prereq.md). + +- `docspell-restserver` this images contains the http server +- `docspell-joex` this image contains the job executor and all + required software (ocrmypdf, unoconv etc) mentioned in + [prerequisites](@/docs/install/prereq.md). +- `docspell-tools` this image simply contains all the scripts from the + `tools/` folder. They are installed into a location in `$PATH`. It + doesn't specify a `CMD` or `ENTRYPOINT`, so you must choose which + script to run. The scripts are all prefixed by `ds-`. So to run the + `consumedir.sh` script, execute `ds-consumedir`. + +### Examples + +These examples use `docker run` to start the restserver and +jobexecutor. Both must be connected to the same database. For this +example, a shared directory is used and the in-process database H2. +For a real setup, using PostgreSQL is recommended. + +This requires to change the default config. This example creates a new +config file. Please refer to the [configuration +page](@/docs/configure/_index.md) for more details. + +``` bash +$ cat > /tmp/docspell.conf <<-"EOF" +# common settings +db_url = "jdbc:h2:///var/docspell/db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" + +# job executor settings +docspell.joex.jdbc.url = ${db_url} +docspell.joex.base-url = "http://10.4.3.2:7878" +docspell.joex.bind.address = "0.0.0.0" + +# restserver settings +docspell.server.backend.jdbc.url = ${db_url} +docspell.server.bind.address = "0.0.0.0" +docspell.server.integration-endpoint { + enabled = true + http-header { + enabled = true + header-value = "test123" + } +} +EOF +``` + +This sets the db url to the same value for both components; thus we +can use the same file for both components. It also sets the bind +address to bind the server socket on all interfaces. Another thing to +note is the `base-url` setting for joex. This publishes joex by this +ip, such that the server component can notify the job executor for new +work. The `integration-endpoint` setting is explained later. + +After creating a common network, we start the rest server and joex: +``` +$ docker network create --subnet 10.4.3.0/24 dsnet +$ docker run -d --name ds-restserver \ + --network dsnet --ip 10.4.3.3 \ + -p 127.0.0.1:7880:7880 \ + -v /tmp/testdb:/var/docspell \ + -v /tmp/docspell.conf:/opt/docspell.conf \ + docspell/restserver:latest /opt/docspell.conf +$ docker run -d --name ds-joex \ + --network dsnet --ip 10.4.3.2 \ + -v /tmp/testdb:/var/docspell \ + -v /tmp/docspell.conf:/opt/docspell.conf \ + docspell/joex:latest /opt/docspell.conf +``` + +After this `docker ps` should show these two containers. Go to +`http://localhost:7880` and sign up/login and start playing around. +When signing up, use the same name for collective and user and then +login with this name. + +For the last part, we use the `docspell/tools` image to create another +container that watches a directory and pushes files to docspell. + +``` bash +$ docker run -d --name ds-consume \ + --network dsnet --ip 10.4.3.4 \ + -v /tmp/inbox:/var/inbox \ + docspell/tools:latest ds-consumedir -imdv --iheader "Docspell-Integration:test123" \ + --path /var/inbox "http://10.4.3.3:7880/api/v1/open/integration/item" +``` + +This starts the [consumedir](@/docs/tools/consumedir.md) script that +watches a directory and uploads arriving files to docspell server. +This requires the value from the `integration-endpoint` setting to be +allowed to upload files. It also requires you to explicitely enable +this: go to *Collective Profile → Settings* and enable the +*Integration Endpoint*. Then create a subdirectory in `/tmp/inbox` +with the name of the *collective* that you registered and place a file +into the `/tmp/inbox/[collective]` directory. The file is pushed to +docspell and processed shortly after. + +To see all available options, run the script with the `--help` option: +``` bash +$ docker run docspell/tools:latest ds-consumedir --help +``` + + +Note that this is just an example and is only to demonstrate how to +use the docker images. For instance, this setup does not provide +fulltext search. For a more sophisticated docker setup, use +appropriate tools, for example `docker-compose` which is explained +below. + +## Docker Compose + +There is a [docker-compose](https://docs.docker.com/compose/) setup +available in the `/docker/docker-compose` folder. This setup is +similiar to the example above, adding fulltext search and a PostgreSQL +database by using just one command. It's only 3 steps to get started: + +### Start Docspell +#### Get the docker-compose files + +Either via cloning the whole repository: + +```bash +$ git clone https://github.com/eikek/docspell +``` + +This downloads all sources. What you need is only one subdirectory. So +if you don't have git or don't want to clone the whole repo, use these +steps instead: + +``` bash +$ mkdir -p docspell/docker/docker-compose +$ cd docspell/docker/docker-compose +$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/docker-compose.yml +$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/docspell.conf +$ wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose/.env +``` + +You can choose any directory instead of +`docspell/docker/docker-compose`, of course. + +Change into the new `docker-compose` directory, for example: + +```bash +$ cd docspell/docker/docker-compose +``` + +#### Run `docker-compose up` + +```bash +$ export DOCSPELL_HEADER_VALUE="my-secret-123" +$ docker-compose up +``` + +The environment variable defines a secret that is shared between some +containers. You can define whatever you like. Please see the +[consumedir.sh](@/docs/tools/consumedir.md#docker) docs for additional +info. + +Goto `http://localhost:7880`, signup and login. When signing up, you +choose the same name for collective and user. Then login with this +name and the password. + +(Optional) Create a folder `./docs/` (the name you +chose for the collective at registration) and place files in there for +importing them. + +The directory contains a file `docspell.conf` that you can +[modify](@/docs/configure/_index.md) as needed. + + +### Override this setup + +If you want to change this setup, you can simply use your own compose +file or add a `docker-compose.override.yml` that allows to amend +certain configs. Look [here](https://docs.docker.com/compose/extends/) +to find more about it. + +As an example, here is a `docker-compose.override.yml`: + +``` yaml +version: '3.7' + +services: + consumedir: + volumes: + - importdocs:/opt/docs + +volumes: + docspell-postgres_data: + driver: local + driver_opts: + type: nfs4 + o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14 + device: ":/mnt/FreeNas/docker_vol1/docspell/postgres_data" + + docspell-solr_data: + driver: local + driver_opts: + type: nfs4 + o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14 + device: ":/mnt/FreeNas/docker_vol1/docspell/solr_data" + + importdocs: + driver: local + driver_opts: + type: nfs4 + o: addr=192.168.x.y,rw,noatime,rsize=8192,wsize=8192,tcp,timeo=14 + device: ":/mnt/FreeNas/archiv/gescannt/output" +``` + + +### Upgrading + +Since [downgrading](@/docs/install/downgrading.md) is not supported, +it is recommended to backup your database before upgrading. Should +something not work as expected, restore the database backup and go +back to the previous version. + +The latest release is always tagged with `latest`. Should you use this +tag, then run these commands to upgrade to newer images: + +``` bash +$ docker-compose down +$ docker-compose pull +$ docker-compose up --force-recreate --build -d +``` diff --git a/website/site/content/docs/install/download_run.md b/website/site/content/docs/install/download_run.md new file mode 100644 index 00000000..5ec930cc --- /dev/null +++ b/website/site/content/docs/install/download_run.md @@ -0,0 +1,121 @@ ++++ +title = "Download & Run" +weight = 22 ++++ + +You can install via zip or deb archives. Please see the +[prerequisites](@/docs/install/prereq.md) first. + +## Using zip files + +You need to download the two files: + +- [docspell-restserver-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver-{{version()}}.zip) +- [docspell-joex-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex-{{version()}}.zip) + + +1. Unzip both files: + ``` bash + $ unzip docspell-*.zip + ``` +2. Open two terminal windows and navigate to the the directory + containing the zip files. +3. Start both components executing: + ``` bash + $ ./docspell-restserver*/bin/docspell-restserver + ``` + in one terminal and + ``` bash + $ ./docspell-joex*/bin/docspell-joex + ``` + in the other. +4. Point your browser to: +5. Register a new account, sign in and try it. + +Note, that this setup doesn't include watching a directory nor +fulltext search. Using zip/deb files requires to take care of the +[prerequisites](@/docs/install/prereq.md) yourself. + +The provided scripts in +[docspell-tools-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-tools-{{version()}}.zip) +must be extracted and installed manually somewhere in your `$PATH`. + +## Using deb files + +Packages are also provided at the release page: + +- [docspell-restserver_{{version()}}_all.deb](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver_{{version()}}_all.deb) +- [docspell-joex_{{version()}}_all.deb](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex_{{version()}}_all.deb) + +The DEB packages can be installed on Debian, or Debian based Distros: + +``` bash +$ sudo dpkg -i docspell*.deb +``` + +Then the start scripts are in your `$PATH`. Run `docspell-restserver` +or `docspell-joex` from a terminal window. + +The packages come with a systemd unit file that will be installed to +autostart the services. + +The provided scripts in `docspell-tools-{{version()}}.zip` must be +extracted and installed manually somewhere in your `$PATH`. There are +no deb files provided. + + +## Running + +Run the start script (in the corresponding `bin/` directory when using +the zip files): + +``` +$ ./docspell-restserver*/bin/docspell-restserver +$ ./docspell-joex*/bin/docspell-joex +``` + +This will startup both components using the default configuration. +Please refer to the [configuration page](@/docs/configure/_index.md) +for how to create a custom config file. Once you have your config +file, simply pass it as argument to the command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf +$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf +``` + +After starting the rest server, you can reach the web application +`http://localhost:7880/`. + +You should be able to create a new account and sign in. When creating +a new account, use the same name for collective and user and then +login with this name. + +## Upgrading + +Since [downgrading](@/docs/install/downgrading.md) is not supported, +it is recommended to backup your database before upgrading. Should +something not work as expected, restore the database backup and go +back to the previous version. + +When using the zip or deb files, either install the new deb files via +your package manager or download and unpack the new zip files. You +might want to have a look at the changelog, since it is sometimes +necessary to modify the config file. + +## More + +### Fulltext Search + +Fulltext search is powered by [SOLR](https://solr.apache.org). You +need to install solr and create a core for docspell. Then cange the +solr url for both components (restserver and joex) accordingly. See +the relevant section in the [config +page](@/docs/configure/_index.md#full-text-search-solr). + + +### Watching a directory + +The [consumedir](@/docs/tools/consumedir.md) script can be used for +this. Using systemd or something similar, it is possible to create a +system service that runs the script in "watch mode". diff --git a/website/site/content/docs/install/installing.md b/website/site/content/docs/install/nix.md similarity index 66% rename from website/site/content/docs/install/installing.md rename to website/site/content/docs/install/nix.md index 192a89dc..a9e44ee4 100644 --- a/website/site/content/docs/install/installing.md +++ b/website/site/content/docs/install/nix.md @@ -1,104 +1,8 @@ +++ -title = "Installing" -weight = 20 +title = "Nix / NixOS" +weight = 24 +++ -# Docker - -There is a [docker-compose](https://docs.docker.com/compose/) setup -available in the `/docker` folder. This setup is also taking care of -all the necessary [prerequisites](@/docs/install/prereq.md) and -creates a container to watch a directory for incoming files. It's only -3 steps: - -1. Clone the github repository - ```bash - $ git clone https://github.com/eikek/docspell - ``` - If you don't have git or don't want to clone the whole repo, use these steps instead: - ``` bash - mkdir -p docspell/docker - cd docspell/docker - wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docker-compose.yml - wget https://raw.githubusercontent.com/eikek/docspell/master/docker/docspell.conf - wget https://raw.githubusercontent.com/eikek/docspell/master/docker/.env - ``` -2. Change into the `docker` directory: - ```bash - $ cd docspell/docker - ``` -3. Run `docker-compose up`: - - ```bash - $ export DOCSPELL_HEADER_VALUE="my-secret-123" - $ docker-compose up - ``` - - The environment variable defines a secret that is shared between - some containers. You can define whatever you like. Please see the - [consumedir.sh](@/docs/tools/consumedir.md#docker) docs for - additional info. -4. Goto `http://localhost:7880`, signup and login. When signing up, - you can choose the same name for collective and user. Then login - with this name and the password. - -5. (Optional) Create a folder `./docs/` (the name you - chose for the collective at registration) and place files in there - for importing them. - -The directory contains a file `docspell.conf` that you can -[modify](@/docs/configure/_index.md) as needed. - -# Download, Unpack, Run - -You can install via zip or deb archives. Please see the -[prerequisites](@/docs/install/prereq.md) first. - -## Using zip files - -You need to download the two files: - -- [docspell-restserver-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver-{{version()}}.zip) -- [docspell-joex-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex-{{version()}}.zip) - - -1. Unzip both files: - ``` bash - $ unzip docspell-*.zip - ``` -2. Open two terminal windows and navigate to the the directory - containing the zip files. -3. Start both components executing: - ``` bash - $ ./docspell-restserver*/bin/docspell-restserver - ``` - in one terminal and - ``` bash - $ ./docspell-joex*/bin/docspell-joex - ``` - in the other. -4. Point your browser to: -5. Register a new account, sign in and try it. - -Note, that this setup doesn't include watching a directory. You can -use the [consumedir.sh](@/docs/tools/consumedir.md) tool for this or -use the docker variant below. - -## Using deb files - -The DEB packages can be installed on Debian, or Debian based Distros: - -``` bash -$ sudo dpkg -i docspell*.deb -``` - -Then the start scripts are in your `$PATH`. Run `docspell-restserver` -or `docspell-joex` from a terminal window. - -The packages come with a systemd unit file that will be installed to -autostart the services. - -# Nix ## Install via Nix @@ -159,6 +63,17 @@ selecting the most current release. For example it translates to is `{{version()}}`. + +## Upgrading + +Since [downgrading](@/docs/install/downgrading.md) is not supported, +it is recommended to backup your database before upgrading. Should +something not work as expected, restore the database backup and go +back to the previous version. + +When using the provided nix setup, the `currentPkg` always points to +the latest release. Thus it is enough to run `nix-build`. + ## Docspell on NixOS {#nixos} If you are running [NixOS](https://nixos.org), there is a module diff --git a/website/site/content/docs/install/prereq.md b/website/site/content/docs/install/prereq.md index dd381f03..9dc438f2 100644 --- a/website/site/content/docs/install/prereq.md +++ b/website/site/content/docs/install/prereq.md @@ -75,7 +75,7 @@ no fulltext search. When installing manually (i.e. not via docker), just install solr and create a core as described in the [solr -documentation](https://lucene.apache.org/solr/guide/8_4/installing-solr.html). +documentation](https://solr.apache.org/guide/8_4/installing-solr.html). That will provide you with the connection url (the last part is the core name). diff --git a/website/site/content/docs/install/quickstart.md b/website/site/content/docs/install/quickstart.md index 0cdd649a..acafd9e4 100644 --- a/website/site/content/docs/install/quickstart.md +++ b/website/site/content/docs/install/quickstart.md @@ -5,28 +5,43 @@ weight = 0 To get started, here are some quick links: -- Using [docker and - docker-compose](@/docs/install/installing.md#docker). This sets up - everything: all prerequisites, both docspell components and a - container running the [consumedir.sh](@/docs/tools/consumedir.md) +- Using [docker and docker-compose](@/docs/install/docker.md). This + sets up everything: all prerequisites, both docspell components and + a container running the [consumedir.sh](@/docs/tools/consumedir.md) script to import files that are dropped in a folder. -- [Download, Unpack and - Run](@/docs/install/installing.md#download-unpack-run). This option - is also very quick, but you need to check the +- [Download, Unpack and Run](@/docs/install/download_run.md). This + option is also very quick, but you need to check the [prerequisites](@/docs/install/prereq.md) yourself. Database is already setup, but you'd need to setup SOLR (when using fulltext search) and install some programs for the joex component. This applies to the `zip` and `deb` files. The files can be downloaded - from the [release page](https://github.com/eikek/docspell/releases/latest). -- via the [nix package manager](@/docs/install/installing.md#nix) and/or as a [NixOS - module](@/docs/install/installing.md#nixos). If you use nix/nixos, you - know what to do. The linked page contains some examples. -- [Unraid](https://www.unraid.net/): While not official, there are - user provided [notes and unraid + from the [release + page](https://github.com/eikek/docspell/releases/latest). +- via the [nix package manager](@/docs/install/nix.md) and/or as a + [NixOS module](@/docs/install/nix.md#nixos). If you use nix/nixos, + you know what to do. The linked page contains some examples. +- [Unraid](https://www.unraid.net/): There are user provided [notes + and unraid templates](https://github.com/vakilando/unraid-docker-templates) which can get you started. Thanks for providing these! -Every [component](@docs/intro/_index.md#components) (restserver, joex, +Every [component](@/docs/intro/_index.md#components) (restserver, joex, consumedir) can run on different machines and multiple times. Most of the time running all on one machine is sufficient and also for simplicity, the docker-compose setup reflects this variant. + +While there are many different ways to run docspell, at some point all +call docspell binaries. These accept one argument: a [config +file](@/docs/configure/_index.md). If this is not given, the default +is used, which gets you started on a single machine, but it is very +likely you want to change these to match your use-case/setup. + +{% infobubble(mode="info", title="⚠ Please note") %} + +Please have a look at the [configuration +page](@/docs/configure/_index.md) page, before making docspell +publicly available. By default, everyone can create an account. This +is great for trying out and using it in an internal network. But when +opened up to the outside, it is recommended to lock this down. + +{% end %} diff --git a/website/site/content/docs/install/running.md b/website/site/content/docs/install/running.md deleted file mode 100644 index 466ef352..00000000 --- a/website/site/content/docs/install/running.md +++ /dev/null @@ -1,93 +0,0 @@ -+++ -title = "Running" -weight = 30 -+++ - -# Running - -Run the start script (in the corresponding `bin/` directory when using -the zip files): - -``` -$ ./docspell-restserver*/bin/docspell-restserver -$ ./docspell-joex*/bin/docspell-joex -``` - -This will startup both components using the default configuration. The -configuration should be adopted to your needs. For example, the -database connection is configured to use a H2 database in the `/tmp` -directory. Please refer to the [configuration -page](@/docs/configure/_index.md) for how to create a custom config -file. Once you have your config file, simply pass it as argument to -the command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf -$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf -``` - -After starting the rest server, you can reach the web application at -path `/app`, so using default values it would be -`http://localhost:7880/app`. There also is a redirect from `/` to -`/app`. - -You should be able to create a new account and sign in. Check the -[configuration page](@/docs/configure/_index.md) to further customize -docspell. - -{% infobubble(mode="info", title="⚠ Please note") %} - -Please have a look at the [configuration -page](@/docs/configure/_index.md) page, before making docspell -available in the internet. By default, everyone can create an account. -This is great for trying out and using it in an internal network. But -when opened up to the outside, it is recommended to lock this down. - -{% end %} - -## Memory - -Using the options below you can define how much memory the JVM process -is able to use. This might be necessary to adopt depending on the -usage scenario and configured text analysis features. - -Please have a look at the corresponding [configuration -section](@/docs/configure/_index.md#memory-usage). - -## Options - -The start scripts support some options to configure the JVM. One often -used setting is the maximum heap size of the JVM. By default, java -determines it based on properties of the current machine. You can -specify it by given java startup options to the command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf -``` - -This would limit the maximum heap to 1GB. The double slash separates -internal options and the arguments to the program. Another frequently -used option is to change the default temp directory. Usually it is -`/tmp`, but it may be desired to have a dedicated temp directory, -which can be configured: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf -``` - -The command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -h -``` - -gives an overview of supported options. - -It is recommended to run joex with the G1GC enabled. If you use java8, -you need to add an option to use G1GC (`-XX:+UseG1GC`), for java11 -this is not necessary (but doesn't hurt either). This could look like -this: - -``` -./docspell-joex-{{version()}}/bin/docspell-joex -J-Xmx1596M -J-XX:+UseG1GC -- /path/to/joex.conf -``` diff --git a/website/site/content/docs/install/upgrading.md b/website/site/content/docs/install/upgrading.md deleted file mode 100644 index 1ae2e8ab..00000000 --- a/website/site/content/docs/install/upgrading.md +++ /dev/null @@ -1,35 +0,0 @@ -+++ -title = "Upgrading" -weight = 35 -+++ - -Upgrading docspell requires to download the newer binaries and -re-installing them depending on your setup. - -The database is migrated on application start automatically. - -Since [downgrading](@/docs/install/downgrading.md) is not supported, -it is recommended to backup your database before upgrading. Should -something not work as expected, restore the database backup and go -back to the previous version. - -# Docker-Compose - -When using the provided `docker-compose` setup, you only need to pull -the new images. The latest release is always tagged with `-LATEST`. - -``` bash -$ docker-compose down -$ docker-compose pull -$ docker-compose up --force-recreate --build -d -``` - -# Zip / Deb Files - -When using the zip or deb files, either install the new deb files via -your package manager or download and unpack the new zip files. - -# Nix - -When using the provided nix setup, the `currentPkg` always points to -the latest release. Thus it is enough to run `nix-build`. diff --git a/website/site/content/docs/intro/_index.md b/website/site/content/docs/intro/_index.md index d14484be..cd394997 100644 --- a/website/site/content/docs/intro/_index.md +++ b/website/site/content/docs/intro/_index.md @@ -85,7 +85,7 @@ database. And it is also recommended (though not strictly required), that all components can reach each other. The fulltext search index is another separate component, where -currently only [SOLR](https://lucene.apache.org/solr) is supported. +currently only [SOLR](https://solr.apache.org) is supported. Fulltext search is optional, so the SOLR component is not required if docspell is run without fulltext search support. diff --git a/website/site/content/docs/webapp/finding.md b/website/site/content/docs/webapp/finding.md index f100d8e2..b39560f6 100644 --- a/website/site/content/docs/webapp/finding.md +++ b/website/site/content/docs/webapp/finding.md @@ -161,10 +161,10 @@ unless one of the following is true: ## The Query The query string for full text search is very powerful. Docspell -currently supports [Apache SOLR](https://lucene.apache.org/solr/) as +currently supports [Apache SOLR](https://solr.apache.org/) as full text search backend, so you may want to have a look at their [documentation on query -syntax](https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing) +syntax](https://solr.apache.org/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing) for a in depth guide. - Wildcards: `?` matches any single character, `*` matches zero or